diff --git a/fastcore/_nbdev.py b/fastcore/_nbdev.py index f97c7ec8..915dec87 100644 --- a/fastcore/_nbdev.py +++ b/fastcore/_nbdev.py @@ -19,136 +19,103 @@ "test_fig_exists": "00_test.ipynb", "ExceptionExpected": "00_test.ipynb", "exception": "00_test.ipynb", - "defaults": "01_foundation.ipynb", - "copy_func": "01_foundation.ipynb", - "patch_to": "01_foundation.ipynb", - "patch": "01_foundation.ipynb", - "patch_property": "01_foundation.ipynb", - "add_docs": "01_foundation.ipynb", - "docs": "01_foundation.ipynb", - "custom_dir": "01_foundation.ipynb", - "arg0": "01_foundation.ipynb", - "arg1": "01_foundation.ipynb", - "arg2": "01_foundation.ipynb", - "arg3": "01_foundation.ipynb", - "arg4": "01_foundation.ipynb", - "coll_repr": "01_foundation.ipynb", - "is_bool": "01_foundation.ipynb", - "mask2idxs": "01_foundation.ipynb", - "cycle": "01_foundation.ipynb", - "zip_cycle": "01_foundation.ipynb", - "is_indexer": "01_foundation.ipynb", - "negate_func": "01_foundation.ipynb", - "GetAttr": "01_foundation.ipynb", - "delegate_attr": "01_foundation.ipynb", - "bind": "01_foundation.ipynb", - "argwhere": "01_foundation.ipynb", - "map_ex": "01_foundation.ipynb", - "filter_ex": "01_foundation.ipynb", - "range_of": "02_utils.ipynb", - "sorted_ex": "01_foundation.ipynb", - "listable_types": "01_foundation.ipynb", - "renumerate": "01_foundation.ipynb", - "first": "01_foundation.ipynb", - "nested_attr": "01_foundation.ipynb", - "stop": "01_foundation.ipynb", - "tst": "01_foundation.ipynb", - "tst2": "01_foundation.ipynb", - "CollBase": "01_foundation.ipynb", - "L": "01_foundation.ipynb", - "L.__signature__": "01_foundation.ipynb", - "save_config_file": "01_foundation.ipynb", - "read_config_file": "01_foundation.ipynb", - "Config": "01_foundation.ipynb", - "ifnone": "02_utils.ipynb", - "maybe_attr": "02_utils.ipynb", - "basic_repr": "02_utils.ipynb", - "get_class": "02_utils.ipynb", - "mk_class": "02_utils.ipynb", - "wrap_class": "02_utils.ipynb", - "ignore_exceptions": "02_utils.ipynb", - "exec_local": "02_utils.ipynb", - "risinstance": "02_utils.ipynb", - "Inf": "02_utils.ipynb", - "in_": "02_utils.ipynb", - "operator.in_": "02_utils.ipynb", - "true": "02_utils.ipynb", - "gen": "02_utils.ipynb", - "chunked": "02_utils.ipynb", - "otherwise": "02_utils.ipynb", - "AttrDict": "02_utils.ipynb", - "dict2obj": "02_utils.ipynb", - "with_cast": "02_utils.ipynb", - "store_attr": "02_utils.ipynb", - "attrdict": "02_utils.ipynb", - "properties": "02_utils.ipynb", - "camel2snake": "02_utils.ipynb", - "snake2camel": "02_utils.ipynb", - "class2attr": "02_utils.ipynb", - "hasattrs": "02_utils.ipynb", - "setattrs": "02_utils.ipynb", - "try_attrs": "02_utils.ipynb", - "ShowPrint": "02_utils.ipynb", - "Int": "02_utils.ipynb", - "Str": "02_utils.ipynb", - "Float": "02_utils.ipynb", - "tuplify": "02_utils.ipynb", - "detuplify": "02_utils.ipynb", - "replicate": "02_utils.ipynb", - "uniqueify": "02_utils.ipynb", - "setify": "02_utils.ipynb", - "merge": "02_utils.ipynb", - "is_listy": "02_utils.ipynb", - "groupby": "02_utils.ipynb", - "last_index": "02_utils.ipynb", - "shufflish": "02_utils.ipynb", - "filter_dict": "02_utils.ipynb", - "filter_keys": "02_utils.ipynb", - "filter_values": "02_utils.ipynb", - "IterLen": "02_utils.ipynb", - "ReindexCollection": "02_utils.ipynb", - "num_methods": "02_utils.ipynb", - "rnum_methods": "02_utils.ipynb", - "inum_methods": "02_utils.ipynb", - "fastuple": "02_utils.ipynb", - "trace": "02_utils.ipynb", - "compose": "02_utils.ipynb", - "maps": "02_utils.ipynb", - "partialler": "02_utils.ipynb", - "mapped": "02_utils.ipynb", - "instantiate": "02_utils.ipynb", - "using_attr": "02_utils.ipynb", - "Self": "02_utils.ipynb", - "Path.readlines": "02_utils.ipynb", - "Path.mk_write": "02_utils.ipynb", - "Path.ls": "02_utils.ipynb", - "open_file": "02_utils.ipynb", - "save_pickle": "02_utils.ipynb", - "load_pickle": "02_utils.ipynb", - "Path.__repr__": "02_utils.ipynb", - "bunzip": "02_utils.ipynb", - "join_path_file": "02_utils.ipynb", - "urlread": "02_utils.ipynb", - "urljson": "02_utils.ipynb", - "run": "02_utils.ipynb", - "do_request": "02_utils.ipynb", - "sort_by_run": "02_utils.ipynb", - "PrettyString": "02_utils.ipynb", - "round_multiple": "02_utils.ipynb", - "even_mults": "02_utils.ipynb", - "num_cpus": "02_utils.ipynb", - "defaults.cpus": "02_utils.ipynb", - "add_props": "02_utils.ipynb", - "ContextManagers": "02_utils.ipynb", - "typed": "02_utils.ipynb", - "str2bool": "02_utils.ipynb", - "set_num_threads": "02_utils.ipynb", - "ProcessPoolExecutor": "02_utils.ipynb", - "ThreadPoolExecutor": "02_utils.ipynb", - "parallel": "02_utils.ipynb", - "run_procs": "02_utils.ipynb", - "parallel_gen": "02_utils.ipynb", - "threaded": "02_utils.ipynb", + "defaults": "01_basics.ipynb", + "ifnone": "01_basics.ipynb", + "maybe_attr": "01_basics.ipynb", + "basic_repr": "01_basics.ipynb", + "is_array": "01_basics.ipynb", + "listify": "01_basics.ipynb", + "get_class": "01_basics.ipynb", + "mk_class": "01_basics.ipynb", + "wrap_class": "01_basics.ipynb", + "ignore_exceptions": "01_basics.ipynb", + "exec_local": "01_basics.ipynb", + "risinstance": "01_basics.ipynb", + "Inf": "01_basics.ipynb", + "in_": "01_basics.ipynb", + "operator.in_": "01_basics.ipynb", + "true": "01_basics.ipynb", + "stop": "01_basics.ipynb", + "gen": "01_basics.ipynb", + "chunked": "01_basics.ipynb", + "otherwise": "01_basics.ipynb", + "AttrDict": "01_basics.ipynb", + "with_cast": "01_basics.ipynb", + "store_attr": "01_basics.ipynb", + "attrdict": "01_basics.ipynb", + "properties": "01_basics.ipynb", + "camel2snake": "01_basics.ipynb", + "snake2camel": "01_basics.ipynb", + "class2attr": "01_basics.ipynb", + "hasattrs": "01_basics.ipynb", + "setattrs": "01_basics.ipynb", + "try_attrs": "01_basics.ipynb", + "ShowPrint": "01_basics.ipynb", + "Int": "01_basics.ipynb", + "Str": "01_basics.ipynb", + "Float": "01_basics.ipynb", + "detuplify": "01_basics.ipynb", + "replicate": "01_basics.ipynb", + "setify": "01_basics.ipynb", + "merge": "01_basics.ipynb", + "range_of": "02_foundation.ipynb", + "groupby": "01_basics.ipynb", + "last_index": "01_basics.ipynb", + "filter_dict": "01_basics.ipynb", + "filter_keys": "01_basics.ipynb", + "filter_values": "01_basics.ipynb", + "cycle": "02_foundation.ipynb", + "zip_cycle": "02_foundation.ipynb", + "sorted_ex": "01_basics.ipynb", + "num_methods": "01_basics.ipynb", + "rnum_methods": "01_basics.ipynb", + "inum_methods": "01_basics.ipynb", + "fastuple": "01_basics.ipynb", + "compose": "01_basics.ipynb", + "maps": "01_basics.ipynb", + "partialler": "01_basics.ipynb", + "instantiate": "01_basics.ipynb", + "using_attr": "01_basics.ipynb", + "Self": "01_basics.ipynb", + "PrettyString": "01_basics.ipynb", + "even_mults": "01_basics.ipynb", + "num_cpus": "01_basics.ipynb", + "defaults.cpus": "01_basics.ipynb", + "add_props": "01_basics.ipynb", + "typed": "01_basics.ipynb", + "copy_func": "02_foundation.ipynb", + "patch_to": "02_foundation.ipynb", + "patch": "02_foundation.ipynb", + "patch_property": "02_foundation.ipynb", + "add_docs": "02_foundation.ipynb", + "docs": "02_foundation.ipynb", + "custom_dir": "02_foundation.ipynb", + "arg0": "02_foundation.ipynb", + "arg1": "02_foundation.ipynb", + "arg2": "02_foundation.ipynb", + "arg3": "02_foundation.ipynb", + "arg4": "02_foundation.ipynb", + "coll_repr": "02_foundation.ipynb", + "is_bool": "02_foundation.ipynb", + "mask2idxs": "02_foundation.ipynb", + "is_indexer": "02_foundation.ipynb", + "negate_func": "02_foundation.ipynb", + "GetAttr": "02_foundation.ipynb", + "delegate_attr": "02_foundation.ipynb", + "bind": "02_foundation.ipynb", + "argwhere": "02_foundation.ipynb", + "map_ex": "02_foundation.ipynb", + "filter_ex": "02_foundation.ipynb", + "listable_types": "02_foundation.ipynb", + "renumerate": "02_foundation.ipynb", + "first": "02_foundation.ipynb", + "nested_attr": "02_foundation.ipynb", + "CollBase": "02_foundation.ipynb", + "L": "02_foundation.ipynb", + "L.__signature__": "02_foundation.ipynb", + "save_config_file": "02_foundation.ipynb", + "read_config_file": "02_foundation.ipynb", + "Config": "02_foundation.ipynb", "type_hints": "03_dispatch.ipynb", "anno_ret": "03_dispatch.ipynb", "lenient_issubclass": "03_dispatch.ipynb", @@ -161,6 +128,40 @@ "retain_type": "03_dispatch.ipynb", "retain_types": "03_dispatch.ipynb", "explode_types": "03_dispatch.ipynb", + "dict2obj": "03_xtras.ipynb", + "tuplify": "03_xtras.ipynb", + "uniqueify": "03_xtras.ipynb", + "is_listy": "03_xtras.ipynb", + "shufflish": "03_xtras.ipynb", + "mapped": "03_xtras.ipynb", + "IterLen": "03_xtras.ipynb", + "ReindexCollection": "03_xtras.ipynb", + "Path.readlines": "03_xtras.ipynb", + "Path.mk_write": "03_xtras.ipynb", + "Path.ls": "03_xtras.ipynb", + "open_file": "03_xtras.ipynb", + "save_pickle": "03_xtras.ipynb", + "load_pickle": "03_xtras.ipynb", + "Path.__repr__": "03_xtras.ipynb", + "bunzip": "03_xtras.ipynb", + "join_path_file": "03_xtras.ipynb", + "urlread": "03_xtras.ipynb", + "urljson": "03_xtras.ipynb", + "run": "03_xtras.ipynb", + "do_request": "03_xtras.ipynb", + "sort_by_run": "03_xtras.ipynb", + "trace": "03_xtras.ipynb", + "round_multiple": "03_xtras.ipynb", + "modified_env": "03_xtras.ipynb", + "ContextManagers": "03_xtras.ipynb", + "str2bool": "03_xtras.ipynb", + "set_num_threads": "03_xtras.ipynb", + "ProcessPoolExecutor": "03_xtras.ipynb", + "ThreadPoolExecutor": "03_xtras.ipynb", + "parallel": "03_xtras.ipynb", + "run_procs": "03_xtras.ipynb", + "parallel_gen": "03_xtras.ipynb", + "threaded": "03_xtras.ipynb", "Transform": "04_transform.ipynb", "InplaceTransform": "04_transform.ipynb", "DisplayedTransform": "04_transform.ipynb", @@ -195,9 +196,10 @@ "call_parse": "07_script.ipynb"} modules = ["test.py", + "basics.py", "foundation.py", - "utils.py", "dispatch.py", + "xtras.py", "transform.py", "logargs.py", "meta.py", diff --git a/fastcore/basics.py b/fastcore/basics.py new file mode 100644 index 00000000..baf7215e --- /dev/null +++ b/fastcore/basics.py @@ -0,0 +1,541 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/01_basics.ipynb (unless otherwise specified). + +__all__ = ['defaults', 'ifnone', 'maybe_attr', 'basic_repr', 'is_array', 'listify', 'get_class', 'mk_class', + 'wrap_class', 'ignore_exceptions', 'exec_local', 'risinstance', 'Inf', 'in_', 'lt', 'gt', 'le', 'ge', 'eq', + 'ne', 'add', 'sub', 'mul', 'truediv', 'is_', 'is_not', 'in_', 'true', 'stop', 'gen', 'chunked', 'otherwise', + 'AttrDict', 'with_cast', 'store_attr', 'attrdict', 'properties', 'camel2snake', 'snake2camel', 'class2attr', + 'hasattrs', 'setattrs', 'try_attrs', 'ShowPrint', 'Int', 'Str', 'Float', 'detuplify', 'replicate', 'setify', + 'merge', 'range_of', 'groupby', 'last_index', 'filter_dict', 'filter_keys', 'filter_values', 'cycle', + 'zip_cycle', 'sorted_ex', 'num_methods', 'rnum_methods', 'inum_methods', 'fastuple', 'compose', 'maps', + 'partialler', 'instantiate', 'using_attr', 'Self', 'Self', 'PrettyString', 'even_mults', 'num_cpus', + 'add_props', 'typed'] + +# Cell +from .imports import * + +# Cell +defaults = SimpleNamespace() + +# Cell +def ifnone(a, b): + "`b` if `a` is None else `a`" + return b if a is None else a + +# Cell +def maybe_attr(o, attr): + "`getattr(o,attr,o)`" + return getattr(o,attr,o) + +# Cell +def basic_repr(flds=None): + if isinstance(flds, str): flds = re.split(', *', flds) + flds = list(flds or []) + def _f(self): + sig = ', '.join(f'{o}={maybe_attr(getattr(self,o), "__name__")}' for o in flds) + return f'{self.__class__.__name__}({sig})' + return _f + +# Cell +def is_array(x): return hasattr(x,'__array__') or hasattr(x,'iloc') + +def listify(o): + if o is None: return [] + if isinstance(o, list): return o + if isinstance(o, str) or is_array(o): return [o] + if is_iter(o): return list(o) + return [o] + +# Cell +def get_class(nm, *fld_names, sup=None, doc=None, funcs=None, **flds): + "Dynamically create a class, optionally inheriting from `sup`, containing `fld_names`" + attrs = {} + for f in fld_names: attrs[f] = None + for f in listify(funcs): attrs[f.__name__] = f + for k,v in flds.items(): attrs[k] = v + sup = ifnone(sup, ()) + if not isinstance(sup, tuple): sup=(sup,) + + def _init(self, *args, **kwargs): + for i,v in enumerate(args): setattr(self, list(attrs.keys())[i], v) + for k,v in kwargs.items(): setattr(self,k,v) + + all_flds = [*fld_names,*flds.keys()] + def _eq(self,b): + return all([getattr(self,k)==getattr(b,k) for k in all_flds]) + + if not sup: attrs['__repr__'] = basic_repr(all_flds) + attrs['__init__'] = _init + attrs['__eq__'] = _eq + res = type(nm, sup, attrs) + if doc is not None: res.__doc__ = doc + return res + +# Cell +def mk_class(nm, *fld_names, sup=None, doc=None, funcs=None, mod=None, **flds): + "Create a class using `get_class` and add to the caller's module" + if mod is None: mod = sys._getframe(1).f_locals + res = get_class(nm, *fld_names, sup=sup, doc=doc, funcs=funcs, **flds) + mod[nm] = res + +# Cell +def wrap_class(nm, *fld_names, sup=None, doc=None, funcs=None, **flds): + "Decorator: makes function a method of a new class `nm` passing parameters to `mk_class`" + def _inner(f): + mk_class(nm, *fld_names, sup=sup, doc=doc, funcs=listify(funcs)+[f], mod=f.__globals__, **flds) + return f + return _inner + +# Cell +class ignore_exceptions: + "Context manager to ignore exceptions" + def __enter__(self): pass + def __exit__(self, *args): return True + +# Cell +def exec_local(code, var_name): + "Call `exec` on `code` and return the var `var_name" + loc = {} + exec(code, globals(), loc) + return loc[var_name] + +# Cell +def risinstance(types, obj=None): + "Curried `isinstance` but with args reversed" + if not obj: return partial(risinstance,types) + return isinstance(obj, types) + +# Cell +#hide +class _InfMeta(type): + @property + def count(self): return itertools.count() + @property + def zeros(self): return itertools.cycle([0]) + @property + def ones(self): return itertools.cycle([1]) + @property + def nones(self): return itertools.cycle([None]) + +# Cell +class Inf(metaclass=_InfMeta): + "Infinite lists" + pass + +# Cell +_dumobj = object() +def _oper(op,a,b=_dumobj): return (lambda o:op(o,a)) if b is _dumobj else op(a,b) + +def _mk_op(nm, mod): + "Create an operator using `oper` and add to the caller's module" + op = getattr(operator,nm) + def _inner(a, b=_dumobj): return _oper(op, a,b) + _inner.__name__ = _inner.__qualname__ = nm + _inner.__doc__ = f'Same as `operator.{nm}`, or returns partial if 1 arg' + mod[nm] = _inner + +# Cell +def in_(x, a): + "`True` if `x in a`" + return x in a + +operator.in_ = in_ + +# Cell +#nbdev_comment _all_ = ['lt','gt','le','ge','eq','ne','add','sub','mul','truediv','is_','is_not','in_'] + +# Cell +for op in ['lt','gt','le','ge','eq','ne','add','sub','mul','truediv','is_','is_not','in_']: _mk_op(op, globals()) + +# Cell +def true(*args, **kwargs): + "Predicate: always `True`" + return True + +# Cell +def stop(e=StopIteration): + "Raises exception `e` (by default `StopException`)" + raise e + +# Cell +def gen(func, seq, cond=true): + "Like `(func(o) for o in seq if cond(func(o)))` but handles `StopIteration`" + return itertools.takewhile(cond, map(func,seq)) + +# Cell +def chunked(it, chunk_sz=None, drop_last=False, n_chunks=None): + "Return batches from iterator `it` of size `chunk_sz` (or return `n_chunks` total)" + assert bool(chunk_sz) ^ bool(n_chunks) + if n_chunks: chunk_sz = math.ceil(len(it)/n_chunks) + if not isinstance(it, Iterator): it = iter(it) + while True: + res = list(itertools.islice(it, chunk_sz)) + if res and (len(res)==chunk_sz or not drop_last): yield res + if len(res) 0 else itertools.cycle([None]) + +# Cell +def zip_cycle(x, *args): + "Like `itertools.zip_longest` but `cycle`s through elements of all but first argument" + return zip(x, *map(cycle,args)) + +# Cell +def sorted_ex(iterable, key=None, reverse=False): + "Like `sorted`, but if key is str use `attrgetter`; if int use `itemgetter`" + if isinstance(key,str): k=lambda o:getattr(o,key,0) + elif isinstance(key,int): k=itemgetter(key) + else: k=key + return sorted(iterable, key=k, reverse=reverse) + +# Cell +num_methods = """ + __add__ __sub__ __mul__ __matmul__ __truediv__ __floordiv__ __mod__ __divmod__ __pow__ + __lshift__ __rshift__ __and__ __xor__ __or__ __neg__ __pos__ __abs__ +""".split() +rnum_methods = """ + __radd__ __rsub__ __rmul__ __rmatmul__ __rtruediv__ __rfloordiv__ __rmod__ __rdivmod__ + __rpow__ __rlshift__ __rrshift__ __rand__ __rxor__ __ror__ +""".split() +inum_methods = """ + __iadd__ __isub__ __imul__ __imatmul__ __itruediv__ + __ifloordiv__ __imod__ __ipow__ __ilshift__ __irshift__ __iand__ __ixor__ __ior__ +""".split() + +# Cell +class fastuple(tuple): + "A `tuple` with elementwise ops and more friendly __init__ behavior" + def __new__(cls, x=None, *rest): + if x is None: x = () + if not isinstance(x,tuple): + if len(rest): x = (x,) + else: + try: x = tuple(iter(x)) + except TypeError: x = (x,) + return super().__new__(cls, x+rest if rest else x) + + def _op(self,op,*args): + if not isinstance(self,fastuple): self = fastuple(self) + return type(self)(map(op,self,*map(cycle, args))) + + def mul(self,*args): + "`*` is already defined in `tuple` for replicating, so use `mul` instead" + return fastuple._op(self, operator.mul,*args) + + def add(self,*args): + "`+` is already defined in `tuple` for concat, so use `add` instead" + return fastuple._op(self, operator.add,*args) + +def _get_op(op): + if isinstance(op,str): op = getattr(operator,op) + def _f(self,*args): return self._op(op,*args) + return _f + +for n in num_methods: + if not hasattr(fastuple, n) and hasattr(operator,n): setattr(fastuple,n,_get_op(n)) + +for n in 'eq ne lt le gt ge'.split(): setattr(fastuple,n,_get_op(n)) +setattr(fastuple,'__invert__',_get_op('__not__')) +setattr(fastuple,'max',_get_op(max)) +setattr(fastuple,'min',_get_op(min)) + +# Cell +def compose(*funcs, order=None): + "Create a function that composes all functions in `funcs`, passing along remaining `*args` and `**kwargs` to all" + funcs = listify(funcs) + if len(funcs)==0: return noop + if len(funcs)==1: return funcs[0] + if order is not None: funcs = sorted_ex(funcs, key=order) + def _inner(x, *args, **kwargs): + for f in funcs: x = f(x, *args, **kwargs) + return x + return _inner + +# Cell +def maps(*args, retain=noop): + "Like `map`, except funcs are composed first" + f = compose(*args[:-1]) + def _f(b): return retain(f(b), b) + return map(_f, args[-1]) + +# Cell +def partialler(f, *args, order=None, **kwargs): + "Like `functools.partial` but also copies over docstring" + fnew = partial(f,*args,**kwargs) + fnew.__doc__ = f.__doc__ + if order is not None: fnew.order=order + elif hasattr(f,'order'): fnew.order=f.order + return fnew + +# Cell +def instantiate(t): + "Instantiate `t` if it's a type, otherwise do nothing" + return t() if isinstance(t, type) else t + +# Cell +def _using_attr(f, attr, x): return f(getattr(x,attr)) + +# Cell +def using_attr(f, attr): + "Change function `f` to operate on `attr`" + return partial(_using_attr, f, attr) + +# Cell +class _Self: + "An alternative to `lambda` for calling methods on passed object." + def __init__(self): self.nms,self.args,self.kwargs,self.ready = [],[],[],True + def __repr__(self): return f'self: {self.nms}({self.args}, {self.kwargs})' + + def __call__(self, *args, **kwargs): + if self.ready: + x = args[0] + for n,a,k in zip(self.nms,self.args,self.kwargs): + x = getattr(x,n) + if callable(x) and a is not None: x = x(*a, **k) + return x + else: + self.args.append(args) + self.kwargs.append(kwargs) + self.ready = True + return self + + def __getattr__(self,k): + if not self.ready: + self.args.append(None) + self.kwargs.append(None) + self.nms.append(k) + self.ready = False + return self + +# Cell +class _SelfCls: + def __getattr__(self,k): return getattr(_Self(),k) + def __getitem__(self,i): return self.__getattr__('__getitem__')(i) + +Self = _SelfCls() + +# Cell +#nbdev_comment _all_ = ['Self'] + +# Cell +class PrettyString(str): + "Little hack to get strings to show properly in Jupyter." + def __repr__(self): return self + +# Cell +def even_mults(start, stop, n): + "Build log-stepped array from `start` to `stop` in `n` steps." + if n==1: return stop + mult = stop/start + step = mult**(1/(n-1)) + return [start*(step**i) for i in range(n)] + +# Cell +def num_cpus(): + "Get number of cpus" + try: return len(os.sched_getaffinity(0)) + except AttributeError: return os.cpu_count() + +defaults.cpus = num_cpus() + +# Cell +def add_props(f, g=None, n=2): + "Create properties passing each of `range(n)` to f" + if g is None: return (property(partial(f,i)) for i in range(n)) + return (property(partial(f,i), partial(g,i)) for i in range(n)) + +# Cell +def _typeerr(arg, val, typ): return TypeError(f"{arg}=={val} not {typ}") + +# Cell +def typed(f): + "Decorator to check param and return types at runtime" + names = f.__code__.co_varnames + anno = f.__annotations__ + ret = anno.pop('return',None) + def _f(*args,**kwargs): + kw = {**kwargs} + if len(anno) > 0: + for i,arg in enumerate(args): kw[names[i]] = arg + for k,v in kw.items(): + if not isinstance(v,anno[k]): raise _typeerr(k, v, anno[k]) + res = f(*args,**kwargs) + if ret is not None and not isinstance(res,ret): raise _typeerr("return", res, ret) + return res + return functools.update_wrapper(_f, f) \ No newline at end of file diff --git a/fastcore/foundation.py b/fastcore/foundation.py index 0d83c5bc..b3c975cb 100644 --- a/fastcore/foundation.py +++ b/fastcore/foundation.py @@ -1,22 +1,19 @@ -# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/01_foundation.ipynb (unless otherwise specified). +# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/02_foundation.ipynb (unless otherwise specified). -__all__ = ['defaults', 'copy_func', 'patch_to', 'patch', 'patch_property', 'add_docs', 'docs', 'custom_dir', 'arg0', - 'arg1', 'arg2', 'arg3', 'arg4', 'coll_repr', 'is_bool', 'mask2idxs', 'cycle', 'zip_cycle', 'is_indexer', - 'negate_func', 'GetAttr', 'delegate_attr', 'bind', 'argwhere', 'map_ex', 'filter_ex', 'range_of', - 'sorted_ex', 'listable_types', 'renumerate', 'first', 'nested_attr', 'stop', 'tst', 'tst2', 'CollBase', 'L', - 'save_config_file', 'read_config_file', 'Config'] +__all__ = ['copy_func', 'patch_to', 'patch', 'patch_property', 'add_docs', 'docs', 'custom_dir', 'arg0', 'arg1', 'arg2', + 'arg3', 'arg4', 'coll_repr', 'is_bool', 'mask2idxs', 'cycle', 'zip_cycle', 'is_indexer', 'negate_func', + 'GetAttr', 'delegate_attr', 'bind', 'argwhere', 'map_ex', 'filter_ex', 'range_of', 'listable_types', + 'renumerate', 'first', 'nested_attr', 'CollBase', 'L', 'save_config_file', 'read_config_file', 'Config'] # Cell from .imports import * +from .basics import * from functools import lru_cache from contextlib import contextmanager from copy import copy from configparser import ConfigParser import random,pickle -# Cell -defaults = SimpleNamespace() - # Cell def copy_func(f): "Copy a non-builtin function (NB `copy.copy` does not work for this)" @@ -113,20 +110,10 @@ def mask2idxs(mask): if is_bool(it): return [i for i,m in enumerate(mask) if m] return [int(i) for i in mask] -# Cell -def _is_array(x): return hasattr(x,'__array__') or hasattr(x,'iloc') - -def _listify(o): - if o is None: return [] - if isinstance(o, list): return o - if isinstance(o, str) or _is_array(o): return [o] - if is_iter(o): return list(o) - return [o] - # Cell def cycle(o): "Like `itertools.cycle` except creates list of `None`s if `o` is empty" - o = _listify(o) + o = listify(o) return itertools.cycle(o) if o is not None and len(o) > 0 else itertools.cycle([None]) # Cell @@ -217,14 +204,6 @@ def range_of(a, b=None, step=None): if is_coll(a): a = len(a) return list(range(a,b,step) if step is not None else range(a,b) if b is not None else range(a)) -# Cell -def sorted_ex(iterable, key=None, reverse=False): - "Like `sorted`, but if key is str use `attrgetter`; if int use `itemgetter`" - if isinstance(key,str): k=lambda o:getattr(o,key,0) - elif isinstance(key,int): k=itemgetter(key) - else: k=key - return sorted(iterable, key=k, reverse=reverse) - # Cell listable_types = typing.Collection,Generator,map,filter,zip @@ -247,23 +226,6 @@ def nested_attr(o, attr, default=None): except AttributeError: return default return o -# Cell -def stop(e=StopIteration): - "Raises exception `e` (by default `StopException`)" - raise e - -def tst(): - try: - stop() - except StopIteration: - return True - -def tst2(): - try: - stop(e=ValueError) - except ValueError: - return True - # Cell class CollBase: "Base class for composing a list of `items`" @@ -288,8 +250,8 @@ class L(GetAttr, CollBase, metaclass=_L_Meta): def __init__(self, items=None, *rest, use_list=False, match=None): if rest: items = (items,)+rest if items is None: items = [] - if (use_list is not None) or not _is_array(items): - items = list(items) if use_list else _listify(items) + if (use_list is not None) or not is_array(items): + items = list(items) if use_list else listify(items) if match is not None: if is_coll(match): match = len(match) if len(items)==1: items = items*match @@ -313,7 +275,7 @@ def __setitem__(self, idx, o): "Set `idx` (can be list of indices, or mask, or int) items to `o` (which is broadcast if not iterable)" if isinstance(idx, int): self.items[idx] = o else: - idx = idx if isinstance(idx,L) else _listify(idx) + idx = idx if isinstance(idx,L) else listify(idx) if not is_iter(o): o = [o]*len(idx) for i,o_ in zip(idx,o): self.items[i] = o_ @@ -329,9 +291,9 @@ def __reversed__(self): return self._new(reversed(self.items)) def __invert__(self): return self._new(not i for i in self) def __repr__(self): return repr(self.items) def _repr_pretty_(self, p, cycle): - p.text('...' if cycle else repr(self.items) if _is_array(self.items) else coll_repr(self)) + p.text('...' if cycle else repr(self.items) if is_array(self.items) else coll_repr(self)) def __mul__ (a,b): return a._new(a.items*b) - def __add__ (a,b): return a._new(a.items+_listify(b)) + def __add__ (a,b): return a._new(a.items+listify(b)) def __radd__(a,b): return a._new(b)+a def __addi__(a,b): a.items += list(b) diff --git a/fastcore/test.py b/fastcore/test.py index 9515793d..16aac98f 100644 --- a/fastcore/test.py +++ b/fastcore/test.py @@ -6,7 +6,6 @@ # Cell from .imports import * -from .utils import * from collections import Counter from contextlib import redirect_stdout @@ -102,7 +101,7 @@ def test_fig_exists(ax): # Cell class ExceptionExpected: "Context manager that tests if an exception is raised" - def __init__(self, ex=Exception, regex=''): store_attr() + def __init__(self, ex=Exception, regex=''): self.ex,self.regex = ex,regex def __enter__(self): pass def __exit__(self, type, value, traceback): if not isinstance(value, self.ex) or (self.regex and not re.search(self.regex, f'{value.args}')): diff --git a/fastcore/utils.py b/fastcore/utils.py index 12154310..a98d8f02 100644 --- a/fastcore/utils.py +++ b/fastcore/utils.py @@ -1,865 +1,5 @@ -# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/02_utils.ipynb (unless otherwise specified). - -__all__ = ['ifnone', 'maybe_attr', 'basic_repr', 'get_class', 'mk_class', 'wrap_class', 'ignore_exceptions', - 'exec_local', 'risinstance', 'Inf', 'in_', 'lt', 'gt', 'le', 'ge', 'eq', 'ne', 'add', 'sub', 'mul', - 'truediv', 'is_', 'is_not', 'in_', 'true', 'gen', 'chunked', 'otherwise', 'AttrDict', 'dict2obj', - 'with_cast', 'store_attr', 'attrdict', 'properties', 'camel2snake', 'snake2camel', 'class2attr', 'hasattrs', - 'setattrs', 'try_attrs', 'ShowPrint', 'Int', 'Str', 'Float', 'tuplify', 'detuplify', 'replicate', - 'uniqueify', 'setify', 'merge', 'is_listy', 'range_of', 'groupby', 'last_index', 'shufflish', 'filter_dict', - 'filter_keys', 'filter_values', 'IterLen', 'ReindexCollection', 'num_methods', 'rnum_methods', - 'inum_methods', 'fastuple', 'trace', 'compose', 'maps', 'partialler', 'mapped', 'instantiate', 'using_attr', - 'Self', 'Self', 'open_file', 'save_pickle', 'load_pickle', 'bunzip', 'join_path_file', 'urlread', 'urljson', - 'run', 'do_request', 'sort_by_run', 'PrettyString', 'round_multiple', 'even_mults', 'num_cpus', 'add_props', - 'ContextManagers', 'typed', 'str2bool', 'set_num_threads', 'ProcessPoolExecutor', 'ThreadPoolExecutor', - 'parallel', 'run_procs', 'parallel_gen', 'threaded'] - -# Cell from .imports import * -from .foundation import * -from functools import wraps - -import mimetypes,bz2,pickle,random,json,urllib,subprocess,shlex,bz2,gzip,distutils.util -from contextlib import contextmanager -from pdb import set_trace -from urllib.request import Request,urlopen -from urllib.error import HTTPError -from urllib.parse import urlencode -from threading import Thread - -# Cell -def ifnone(a, b): - "`b` if `a` is None else `a`" - return b if a is None else a - -# Cell -def maybe_attr(o, attr): - "`getattr(o,attr,o)`" - return getattr(o,attr,o) - -# Cell -def basic_repr(flds=None): - if isinstance(flds, str): flds = re.split(', *', flds) - flds = L(flds) - def _f(self): - sig = ', '.join(f'{o}={maybe_attr(getattr(self,o), "__name__")}' for o in flds) - return f'{self.__class__.__name__}({sig})' - return _f - -# Cell -def get_class(nm, *fld_names, sup=None, doc=None, funcs=None, **flds): - "Dynamically create a class, optionally inheriting from `sup`, containing `fld_names`" - attrs = {} - for f in fld_names: attrs[f] = None - for f in L(funcs): attrs[f.__name__] = f - for k,v in flds.items(): attrs[k] = v - sup = ifnone(sup, ()) - if not isinstance(sup, tuple): sup=(sup,) - - def _init(self, *args, **kwargs): - for i,v in enumerate(args): setattr(self, list(attrs.keys())[i], v) - for k,v in kwargs.items(): setattr(self,k,v) - - all_flds = [*fld_names,*flds.keys()] - def _eq(self,b): - return all([getattr(self,k)==getattr(b,k) for k in all_flds]) - - if not sup: attrs['__repr__'] = basic_repr(all_flds) - attrs['__init__'] = _init - attrs['__eq__'] = _eq - res = type(nm, sup, attrs) - if doc is not None: res.__doc__ = doc - return res - -# Cell -def mk_class(nm, *fld_names, sup=None, doc=None, funcs=None, mod=None, **flds): - "Create a class using `get_class` and add to the caller's module" - if mod is None: mod = sys._getframe(1).f_locals - res = get_class(nm, *fld_names, sup=sup, doc=doc, funcs=funcs, **flds) - mod[nm] = res - -# Cell -def wrap_class(nm, *fld_names, sup=None, doc=None, funcs=None, **flds): - "Decorator: makes function a method of a new class `nm` passing parameters to `mk_class`" - def _inner(f): - mk_class(nm, *fld_names, sup=sup, doc=doc, funcs=L(funcs)+f, mod=f.__globals__, **flds) - return f - return _inner - -# Cell -class ignore_exceptions: - "Context manager to ignore exceptions" - def __enter__(self): pass - def __exit__(self, *args): return True - -# Cell -def exec_local(code, var_name): - "Call `exec` on `code` and return the var `var_name" - loc = {} - exec(code, globals(), loc) - return loc[var_name] - -# Cell -def risinstance(types, obj=None): - "Curried `isinstance` but with args reversed" - if not obj: return partial(risinstance,types) - return isinstance(obj, types) - -# Cell -#hide -class _InfMeta(type): - @property - def count(self): return itertools.count() - @property - def zeros(self): return itertools.cycle([0]) - @property - def ones(self): return itertools.cycle([1]) - @property - def nones(self): return itertools.cycle([None]) - -# Cell -class Inf(metaclass=_InfMeta): - "Infinite lists" - pass - -# Cell -_dumobj = object() -def _oper(op,a,b=_dumobj): return (lambda o:op(o,a)) if b is _dumobj else op(a,b) - -def _mk_op(nm, mod): - "Create an operator using `oper` and add to the caller's module" - op = getattr(operator,nm) - def _inner(a, b=_dumobj): return _oper(op, a,b) - _inner.__name__ = _inner.__qualname__ = nm - _inner.__doc__ = f'Same as `operator.{nm}`, or returns partial if 1 arg' - mod[nm] = _inner - -# Cell -def in_(x, a): - "`True` if `x in a`" - return x in a - -operator.in_ = in_ - -# Cell -#nbdev_comment _all_ = ['lt','gt','le','ge','eq','ne','add','sub','mul','truediv','is_','is_not','in_'] - -# Cell -for op in ['lt','gt','le','ge','eq','ne','add','sub','mul','truediv','is_','is_not','in_']: _mk_op(op, globals()) - -# Cell -def true(*args, **kwargs): - "Predicate: always `True`" - return True - -# Cell -def gen(func, seq, cond=true): - "Like `(func(o) for o in seq if cond(func(o)))` but handles `StopIteration`" - return itertools.takewhile(cond, map(func,seq)) - -# Cell -def chunked(it, chunk_sz=None, drop_last=False, n_chunks=None): - "Return batches from iterator `it` of size `chunk_sz` (or return `n_chunks` total)" - assert bool(chunk_sz) ^ bool(n_chunks) - if n_chunks: chunk_sz = math.ceil(len(it)/n_chunks) - if not isinstance(it, Iterator): it = iter(it) - while True: - res = list(itertools.islice(it, chunk_sz)) - if res and (len(res)==chunk_sz or not drop_last): yield res - if len(res) 0: - for i,arg in enumerate(args): kw[names[i]] = arg - for k,v in kw.items(): - if not isinstance(v,anno[k]): raise _typeerr(k, v, anno[k]) - res = f(*args,**kwargs) - if ret is not None and not isinstance(res,ret): raise _typeerr("return", res, ret) - return res - return functools.update_wrapper(_f, f) - -# Cell -def str2bool(s): - "Case-insensitive convert string `s` too a bool (`y`,`yes`,`t`,`true`,`on`,`1`->`True`)" - if not isinstance(s,str): return bool(s) - return bool(distutils.util.strtobool(s)) if s else False - -# Cell -from multiprocessing import Process, Queue -import concurrent.futures -import time -from multiprocessing import Manager - -# Cell -def set_num_threads(nt): - "Get numpy (and others) to use `nt` threads" - try: import mkl; mkl.set_num_threads(nt) - except: pass - try: import torch; torch.set_num_threads(nt) - except: pass - os.environ['IPC_ENABLE']='1' - for o in ['OPENBLAS_NUM_THREADS','NUMEXPR_NUM_THREADS','OMP_NUM_THREADS','MKL_NUM_THREADS']: - os.environ[o] = str(nt) - -# Cell -def _call(lock, pause, n, g, item): - l = False - if pause: - try: - l = lock.acquire(timeout=pause*(n+2)) - time.sleep(pause) - finally: - if l: lock.release() - return g(item) - -# Cell -class ProcessPoolExecutor(concurrent.futures.ProcessPoolExecutor): - "Same as Python's ProcessPoolExecutor, except can pass `max_workers==0` for serial execution" - def __init__(self, max_workers=defaults.cpus, on_exc=print, pause=0, **kwargs): - if max_workers is None: max_workers=defaults.cpus - store_attr() - self.not_parallel = max_workers==0 - if self.not_parallel: max_workers=1 - super().__init__(max_workers, **kwargs) - - def map(self, f, items, timeout=None, chunksize=1, *args, **kwargs): - self.lock = Manager().Lock() - g = partial(f, *args, **kwargs) - if self.not_parallel: return map(g, items) - _g = partial(_call, self.lock, self.pause, self.max_workers, g) - try: return super().map(_g, items, timeout=timeout, chunksize=chunksize) - except Exception as e: self.on_exc(e) - -# Cell -class ThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor): - "Same as Python's ThreadPoolExecutor, except can pass `max_workers==0` for serial execution" - def __init__(self, max_workers=defaults.cpus, on_exc=print, pause=0, **kwargs): - if max_workers is None: max_workers=defaults.cpus - store_attr() - self.not_parallel = max_workers==0 - if self.not_parallel: max_workers=1 - super().__init__(max_workers, **kwargs) - - def map(self, f, items, timeout=None, chunksize=1, *args, **kwargs): - self.lock = Manager().Lock() - g = partial(f, *args, **kwargs) - if self.not_parallel: return map(g, items) - _g = partial(_call, self.lock, self.pause, self.max_workers, g) - try: return super().map(_g, items, timeout=timeout, chunksize=chunksize) - except Exception as e: self.on_exc(e) - -# Cell -try: from fastprogress import progress_bar -except: progress_bar = None - -# Cell -def parallel(f, items, *args, n_workers=defaults.cpus, total=None, progress=None, pause=0, - threadpool=False, timeout=None, chunksize=1, **kwargs): - "Applies `func` in parallel to `items`, using `n_workers`" - if progress is None: progress = progress_bar is not None - pool = ThreadPoolExecutor if threadpool else ProcessPoolExecutor - with pool(n_workers, pause=pause) as ex: - r = ex.map(f,items, *args, timeout=timeout, chunksize=chunksize, **kwargs) - if progress: - if total is None: total = len(items) - r = progress_bar(r, total=total, leave=False) - return L(r) - -# Cell -def run_procs(f, f_done, args): - "Call `f` for each item in `args` in parallel, yielding `f_done`" - processes = L(args).map(Process, args=arg0, target=f) - for o in processes: o.start() - yield from f_done() - processes.map(Self.join()) - -# Cell -def _f_pg(obj, queue, batch, start_idx): - for i,b in enumerate(obj(batch)): queue.put((start_idx+i,b)) - -def _done_pg(queue, items): return (queue.get() for _ in items) - -# Cell -def parallel_gen(cls, items, n_workers=defaults.cpus, **kwargs): - "Instantiate `cls` in `n_workers` procs & call each on a subset of `items` in parallel." - if n_workers==0: - yield from enumerate(list(cls(**kwargs)(items))) - return - batches = L(chunked(items, n_chunks=n_workers)) - idx = L(itertools.accumulate(0 + batches.map(len))) - queue = Queue() - if progress_bar: items = progress_bar(items, leave=False) - f=partial(_f_pg, cls(**kwargs), queue) - done=partial(_done_pg, queue, items) - yield from run_procs(f, done, L(batches,idx).zip()) +from .test import * +from .basics import * +from .xtras import * -# Cell -def threaded(f): - "Run `f` in a thread, and returns the thread" - @wraps(f) - def _f(*args, **kwargs): - res = Thread(target=f, args=args, kwargs=kwargs) - res.start() - return res - return _f \ No newline at end of file diff --git a/fastcore/xtras.py b/fastcore/xtras.py new file mode 100644 index 00000000..84aecf72 --- /dev/null +++ b/fastcore/xtras.py @@ -0,0 +1,382 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/03_xtras.ipynb (unless otherwise specified). + +__all__ = ['dict2obj', 'tuplify', 'uniqueify', 'is_listy', 'shufflish', 'mapped', 'IterLen', 'ReindexCollection', + 'open_file', 'save_pickle', 'load_pickle', 'bunzip', 'join_path_file', 'urlread', 'urljson', 'run', + 'do_request', 'sort_by_run', 'trace', 'round_multiple', 'modified_env', 'ContextManagers', 'str2bool', + 'set_num_threads', 'ProcessPoolExecutor', 'ThreadPoolExecutor', 'parallel', 'run_procs', 'parallel_gen', + 'threaded'] + +# Cell +from .imports import * +from .foundation import * +from .basics import * +from functools import wraps + +import mimetypes,bz2,pickle,random,json,urllib,subprocess,shlex,bz2,gzip,distutils.util +from contextlib import contextmanager,ExitStack +from pdb import set_trace +from urllib.request import Request,urlopen +from urllib.error import HTTPError +from urllib.parse import urlencode +from threading import Thread + +# Cell +def dict2obj(d): + "Convert (possibly nested) dicts (or lists of dicts) to `AttrDict`" + if isinstance(d, (L,list)): return L(d).map(dict2obj) + if not isinstance(d, dict): return d + return AttrDict(**{k:dict2obj(v) for k,v in d.items()}) + +# Cell +def tuplify(o, use_list=False, match=None): + "Make `o` a tuple" + return tuple(L(o, use_list=use_list, match=match)) + +# Cell +def uniqueify(x, sort=False, bidir=False, start=None): + "Unique elements in `x`, optionally `sort`-ed, optionally return reverse correspondence, optionally prepend with elements." + res = L(x).unique() + if start is not None: res = start+res + if sort: res.sort() + if bidir: return res, res.val2idx() + return res + +# Cell +def is_listy(x): + "`isinstance(x, (tuple,list,L,slice,Generator))`" + return isinstance(x, (tuple,list,L,slice,Generator)) + +# Cell +def shufflish(x, pct=0.04): + "Randomly relocate items of `x` up to `pct` of `len(x)` from their starting location" + n = len(x) + return L(x[i] for i in sorted(range_of(x), key=lambda o: o+n*(1+random.random()*pct))) + +# Cell +def mapped(f, it): + "map `f` over `it`, unless it's not listy, in which case return `f(it)`" + return L(it).map(f) if is_listy(it) else f(it) + +# Cell +#hide +class IterLen: + "Base class to add iteration to anything supporting `__len__` and `__getitem__`" + def __iter__(self): return (self[i] for i in range_of(self)) + +# Cell +@docs +class ReindexCollection(GetAttr, IterLen): + "Reindexes collection `coll` with indices `idxs` and optional LRU cache of size `cache`" + _default='coll' + def __init__(self, coll, idxs=None, cache=None, tfm=noop): + if idxs is None: idxs = L.range(coll) + store_attr() + if cache is not None: self._get = functools.lru_cache(maxsize=cache)(self._get) + + def _get(self, i): return self.tfm(self.coll[i]) + def __getitem__(self, i): return self._get(self.idxs[i]) + def __len__(self): return len(self.coll) + def reindex(self, idxs): self.idxs = idxs + def shuffle(self): random.shuffle(self.idxs) + def cache_clear(self): self._get.cache_clear() + def __getstate__(self): return {'coll': self.coll, 'idxs': self.idxs, 'cache': self.cache, 'tfm': self.tfm} + def __setstate__(self, s): self.coll,self.idxs,self.cache,self.tfm = s['coll'],s['idxs'],s['cache'],s['tfm'] + + _docs = dict(reindex="Replace `self.idxs` with idxs", + shuffle="Randomly shuffle indices", + cache_clear="Clear LRU cache") + +# Cell +@patch +def readlines(self:Path, hint=-1, encoding='utf8'): + "Read the content of `self`" + with self.open(encoding=encoding) as f: return f.readlines(hint) + +# Cell +@patch +def mk_write(self:Path, data, encoding=None, errors=None, mode=511): + "Make all parent dirs of `self`" + self.parent.mkdir(exist_ok=True, parents=True, mode=mode) + self.write_text(data, encoding=encoding, errors=errors) + +# Cell +@patch +def ls(self:Path, n_max=None, file_type=None, file_exts=None): + "Contents of path as a list" + extns=L(file_exts) + if file_type: extns += L(k for k,v in mimetypes.types_map.items() if v.startswith(file_type+'/')) + has_extns = len(extns)==0 + res = (o for o in self.iterdir() if has_extns or o.suffix in extns) + if n_max is not None: res = itertools.islice(res, n_max) + return L(res) + +# Cell +def open_file(fn, mode='r'): + "Open a file, with optional compression if gz or bz2 suffix" + if isinstance(fn, io.IOBase): return fn + fn = Path(fn) + if fn.suffix=='.bz2': return bz2.BZ2File(fn, mode) + elif fn.suffix=='.gz' : return gzip.GzipFile(fn, mode) + else: return open(fn,mode) + +# Cell +def save_pickle(fn, o): + "Save a pickle file, to a file name or opened file" + with open_file(fn, 'wb') as f: pickle.dump(o, f) + +# Cell +def load_pickle(fn): + "Load a pickle file from a file name or opened file" + with open_file(fn, 'rb') as f: return pickle.load(f) + +# Cell +@patch +def __repr__(self:Path): + b = getattr(Path, 'BASE_PATH', None) + if b: + try: self = self.relative_to(b) + except: pass + return f"Path({self.as_posix()!r})" + +# Cell +def bunzip(fn): + "bunzip `fn`, raising exception if output already exists" + fn = Path(fn) + assert fn.exists(), f"{fn} doesn't exist" + out_fn = fn.with_suffix('') + assert not out_fn.exists(), f"{out_fn} already exists" + with bz2.BZ2File(fn, 'rb') as src, out_fn.open('wb') as dst: + for d in iter(lambda: src.read(1024*1024), b''): dst.write(d) + +# Cell +def join_path_file(file, path, ext=''): + "Return `path/file` if file is a string or a `Path`, file otherwise" + if not isinstance(file, (str, Path)): return file + path.mkdir(parents=True, exist_ok=True) + return path/f'{file}{ext}' + +# Cell +def urlread(url, data=None, **kwargs): + "Retrieve `url`, using `data` dict or `kwargs` to `POST` if present" + if kwargs and not data: data=kwargs + if data is not None: + if not isinstance(data, (str,bytes)): data = urlencode(data) + if not isinstance(data, bytes): data = data.encode('ascii') + cls = urllib.request.Request + if not isinstance(url,cls): url = cls(url) + url.headers['User-Agent'] = 'Mozilla/5.0' + with urlopen(url, data=data) as res: return res.read() + +# Cell +def urljson(url, data=None): + "Retrieve `url` and decode json" + return json.loads(urlread(url, data=data)) + +# Cell +def run(cmd, *rest, ignore_ex=False, as_bytes=False): + "Pass `cmd` (splitting with `shlex` if string) to `subprocess.run`; return `stdout`; raise `IOError` if fails" + if rest: cmd = (cmd,)+rest + elif isinstance(cmd,str): cmd = shlex.split(cmd) + res = subprocess.run(cmd, capture_output=True) + stdout = res.stdout + if not as_bytes: stdout = stdout.decode() + if ignore_ex: return (res.returncode, stdout) + if res.returncode: raise IOError("{} ;; {}".format(res.stdout, res.stderr)) + return stdout + +# Cell +def do_request(url, post=False, headers=None, **data): + "Call GET or json-encoded POST on `url`, depending on `post`" + if data: + if post: data = json.dumps(data).encode('ascii') + else: + url += "?" + urlencode(data) + data = None + return urljson(Request(url, headers=headers, data=data or None)) + +# Cell +def _is_instance(f, gs): + tst = [g if type(g) in [type, 'function'] else g.__class__ for g in gs] + for g in tst: + if isinstance(f, g) or f==g: return True + return False + +def _is_first(f, gs): + for o in L(getattr(f, 'run_after', None)): + if _is_instance(o, gs): return False + for g in gs: + if _is_instance(f, L(getattr(g, 'run_before', None))): return False + return True + +def sort_by_run(fs): + end = L(fs).attrgot('toward_end') + inp,res = L(fs)[~end] + L(fs)[end], L() + while len(inp): + for i,o in enumerate(inp): + if _is_first(o, inp): + res.append(inp.pop(i)) + break + else: raise Exception("Impossible to sort") + return res + +# Cell +def trace(f): + "Add `set_trace` to an existing function `f`" + if getattr(f, '_traced', False): return f + def _inner(*args,**kwargs): + set_trace() + return f(*args,**kwargs) + _inner._traced = True + return _inner + +# Cell +def round_multiple(x, mult, round_down=False): + "Round `x` to nearest multiple of `mult`" + def _f(x_): return (int if round_down else round)(x_/mult)*mult + res = L(x).map(_f) + return res if is_listy(x) else res[0] + +# Cell +@contextmanager +def modified_env(*delete, **replace): + "Context manager temporarily modifying `os.environ` by deleting `delete` and replacing `replace`" + prev = dict(os.environ) + try: + os.environ.update(replace) + for k in delete: os.environ.pop(k, None) + yield + finally: + os.environ.clear() + os.environ.update(prev) + +# Cell +class ContextManagers(GetAttr): + "Wrapper for `contextlib.ExitStack` which enters a collection of context managers" + def __init__(self, mgrs): self.default,self.stack = L(mgrs),ExitStack() + def __enter__(self): self.default.map(self.stack.enter_context) + def __exit__(self, *args, **kwargs): self.stack.__exit__(*args, **kwargs) + +# Cell +def str2bool(s): + "Case-insensitive convert string `s` too a bool (`y`,`yes`,`t`,`true`,`on`,`1`->`True`)" + if not isinstance(s,str): return bool(s) + return bool(distutils.util.strtobool(s)) if s else False + +# Cell +from multiprocessing import Process, Queue +import concurrent.futures +import time +from multiprocessing import Manager + +# Cell +def set_num_threads(nt): + "Get numpy (and others) to use `nt` threads" + try: import mkl; mkl.set_num_threads(nt) + except: pass + try: import torch; torch.set_num_threads(nt) + except: pass + os.environ['IPC_ENABLE']='1' + for o in ['OPENBLAS_NUM_THREADS','NUMEXPR_NUM_THREADS','OMP_NUM_THREADS','MKL_NUM_THREADS']: + os.environ[o] = str(nt) + +# Cell +def _call(lock, pause, n, g, item): + l = False + if pause: + try: + l = lock.acquire(timeout=pause*(n+2)) + time.sleep(pause) + finally: + if l: lock.release() + return g(item) + +# Cell +class ProcessPoolExecutor(concurrent.futures.ProcessPoolExecutor): + "Same as Python's ProcessPoolExecutor, except can pass `max_workers==0` for serial execution" + def __init__(self, max_workers=defaults.cpus, on_exc=print, pause=0, **kwargs): + if max_workers is None: max_workers=defaults.cpus + store_attr() + self.not_parallel = max_workers==0 + if self.not_parallel: max_workers=1 + super().__init__(max_workers, **kwargs) + + def map(self, f, items, timeout=None, chunksize=1, *args, **kwargs): + self.lock = Manager().Lock() + g = partial(f, *args, **kwargs) + if self.not_parallel: return map(g, items) + _g = partial(_call, self.lock, self.pause, self.max_workers, g) + try: return super().map(_g, items, timeout=timeout, chunksize=chunksize) + except Exception as e: self.on_exc(e) + +# Cell +class ThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor): + "Same as Python's ThreadPoolExecutor, except can pass `max_workers==0` for serial execution" + def __init__(self, max_workers=defaults.cpus, on_exc=print, pause=0, **kwargs): + if max_workers is None: max_workers=defaults.cpus + store_attr() + self.not_parallel = max_workers==0 + if self.not_parallel: max_workers=1 + super().__init__(max_workers, **kwargs) + + def map(self, f, items, timeout=None, chunksize=1, *args, **kwargs): + self.lock = Manager().Lock() + g = partial(f, *args, **kwargs) + if self.not_parallel: return map(g, items) + _g = partial(_call, self.lock, self.pause, self.max_workers, g) + try: return super().map(_g, items, timeout=timeout, chunksize=chunksize) + except Exception as e: self.on_exc(e) + +# Cell +try: from fastprogress import progress_bar +except: progress_bar = None + +# Cell +def parallel(f, items, *args, n_workers=defaults.cpus, total=None, progress=None, pause=0, + threadpool=False, timeout=None, chunksize=1, **kwargs): + "Applies `func` in parallel to `items`, using `n_workers`" + if progress is None: progress = progress_bar is not None + pool = ThreadPoolExecutor if threadpool else ProcessPoolExecutor + with pool(n_workers, pause=pause) as ex: + r = ex.map(f,items, *args, timeout=timeout, chunksize=chunksize, **kwargs) + if progress: + if total is None: total = len(items) + r = progress_bar(r, total=total, leave=False) + return L(r) + +# Cell +def run_procs(f, f_done, args): + "Call `f` for each item in `args` in parallel, yielding `f_done`" + processes = L(args).map(Process, args=arg0, target=f) + for o in processes: o.start() + yield from f_done() + processes.map(Self.join()) + +# Cell +def _f_pg(obj, queue, batch, start_idx): + for i,b in enumerate(obj(batch)): queue.put((start_idx+i,b)) + +def _done_pg(queue, items): return (queue.get() for _ in items) + +# Cell +def parallel_gen(cls, items, n_workers=defaults.cpus, **kwargs): + "Instantiate `cls` in `n_workers` procs & call each on a subset of `items` in parallel." + if n_workers==0: + yield from enumerate(list(cls(**kwargs)(items))) + return + batches = L(chunked(items, n_chunks=n_workers)) + idx = L(itertools.accumulate(0 + batches.map(len))) + queue = Queue() + if progress_bar: items = progress_bar(items, leave=False) + f=partial(_f_pg, cls(**kwargs), queue) + done=partial(_done_pg, queue, items) + yield from run_procs(f, done, L(batches,idx).zip()) + +# Cell +def threaded(f): + "Run `f` in a thread, and returns the thread" + @wraps(f) + def _f(*args, **kwargs): + res = Thread(target=f, args=args, kwargs=kwargs) + res.start() + return res + return _f \ No newline at end of file diff --git a/nbs/00_test.ipynb b/nbs/00_test.ipynb index 08ba6b70..fd5d0190 100644 --- a/nbs/00_test.ipynb +++ b/nbs/00_test.ipynb @@ -17,7 +17,6 @@ "source": [ "#export\n", "from fastcore.imports import *\n", - "from fastcore.utils import *\n", "from collections import Counter\n", "from contextlib import redirect_stdout" ] @@ -530,7 +529,7 @@ "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAIAAABMXPacAACCwklEQVR4nET916+tW3YfiI0xZv7CCjucdFPdqmJFFilSpijJoqBuu200ZLjRkA00/OQHP/nF/4H/ET/YMNoPtgW4G7blbuVIScysKrLIYrHSvefcc84OK31hxjH8sIv268LGwtprfXOEX5r47/7t74rA+fiOlGFQc8xD8N7qTz790t3du2Veak0hDOO4IwXTZXo8PAi34EZG6jt3fbN3zk2X9fd+/3fu7t4qRd4O5/Ppt//tv/j93/uPUjmEzvhOk3JG7ze7b3zr6/+r/83/9ub25vXnnx/Py6uXz6/32/1+31o7nh6n6dx3G+uM1jrGxTl/Ok0grI1a5nmZL12/7fs+dG4ct8aYzz//nBs656bL4+Hxzljb+WDCuN/vjVHW2MfDIxFa6+K6aOPWWKxWpZTWmjZGhE6nh3WZvR+s1fv91Wef//TZ7Yta27IspcjheHj27IU1uN9vjTGN6/t3d+fLyftBYfO++4sf/Wi73cWYjKUuDNM8Aypn3eN5XuaTc3Yz7obO1xKFldIwz+v793eI8vz5i+PxeP940q3CNN+fDm9evPqKceNpebh7/8UHz68RqO/76XIJYWOdLSVTa8MwGGNLTufz8frmdrPZOmdZmBS8evmB92673VrjEemjjz7a7K/+9b/4p6kUbUVASilLXL94/cUP/uSP/8bf/LvT5chNee83m1Epdf9wDwD7/T6usyIHAMZYrY21tpZcq1TRV/ubYTMqbbQma826Rmbo+67WWPLqfNf14zCMXd95H4SFhY1RIpBSIqVjzJdpfXazjZFjzDvvjDXrqo/HHAICYi556Mf7h7sXz1+llEJwIs0Y7PpgrEbAy+V8Pl+M8YpERAPiy1evTqdDSnFecteNuSSlbcyokL1BBM4lnWYFNc3zvN8N3jutdcppWS9EaDXq1mpMyfoNKOecfXV7vXaqG0fmaowlpNZ4ni/WWGus1iDcuj4MYz+OPSKVWkoutbbr61vrjCL0wRPpYfhG6Mac67/5l/+0lEi2I4GU8/F4/tEPvv+1r39zv9/d3r4chs4YfTweT6fzdrMJoW+1TtNpu7tiViklazWhKK2HcYNSBWCN8eZ6L8Ln89lZV2ttDON4Bdo4q7uus9bWWktJRESIKadSQCklwkNniDDXtsa1y7ZW2e1urAnTNIPAPE1d1z8eHnPOSqlay+3tdT/0RAgAj4eH4/HoQ+g7l1NZYgSQoe/Pp9MwDLUWBAjOI1EIIStYMYNySOb+dDGYx6Hb729Titaa1riWNs/rdDlQKst+d3N18/zzL17/+Y9+yK0U0Y1Vrc0aiyStFRH23qdc5nmpNXd92F/tADHlFNc4L+tyPjkNz5+96IfBWuec7bruk08+/c2//fc++OjjlCNLY5GcyxTXH/3ZX3zvj//g5uZ2sx1IYS757u4ORFqr8zyPm51SKsZFa02kam0+dA00sMxrWZZMZIyx0zQt82KMFhClXOi3fRf6vjfGiHCMKyLmHGutrcllOjGzUuScbY0FJJcUU06slpRDF7TWteWUUsrx+vr6dD5437G03W7s+4AIp9PxfJ6cG5xzStvGKAKtNWvtfn/FUj744BOltHfeGrMsZ9LU933KsbUaNN3udx+9+sAYdTi+t1Y5Z0tF710/XlHfDYqw9+F6u3PapALHSxMAJLDWKqUQoe83AOC83e22z1++9D601i6XyzQtCEopxdAYOHS2C51zTivduDK3D16++rt/9+8Za7llAmzCLHI+Tz/9ix96H7wPiHR4PGijdrstSKulCMtuf1tybjUZY73zpfHD43la4p/86LOYy9VuFJF1jc6HZVlqidYqAe664J1n5pSSUqS1FkYBGjab4J2xSimllBaR7dCNYzf0wxhUTZG59X3vXBjHjSKz311prWst4zAYa2qt5/NxWdauGwQorinHOI7Dbr8XpFxbCJ6QmmTrwhrXUuoyrwBoXN+HYY355fP9Bx+87IfudD6sSyJFPtickwhuNyM577Uk5rLZbD755BPn9ctn43YzCAMA+hAQkVCFLtzc7Iex01qXUqZ55obWhNpq3w9C7vMv3nLj0HnnHCk0mpzXX/nyl/7Ob/0nX/2lr5WcKrPWhgFSbfdv306Xk1Z6nueY1mHohmFAYKXUvMwIcnV1k3IsZVWauFWn5PHhvsS4247W2XmeAdE5X2pe5pm5aE1a68at1qy1ZubWmlKaUDtrX7x86b13zolIa01rdXN93YDmNSuieZ6cs4oUM4tAznm73S7rxdpQSru/v1vmqJQympy12iit9WVZSq3SWsrrPF+22/27t58DgAuh6/xudy3S1pj6Yby6uu77johOp3NOdbu9EZah7xHh4Xicl7NWGsF15zlaa1JasVWnVC6p1mqd6fv+cpmfSgqilJKXZa21amWIVGspV6ZctuNwPj+uMW63Y0pJBJBESlOKPvro4//87/+X/4ef/TzlZKxrzLnVu7v7P/neH3XDOM/rOG5rLalk1K4xEECu3Hd+HHcPD/ddaATU6vr65z+6e/f2D+XQbzY5pf3++uWrj533XfBd1znnEaGUZIwVYRFkBhbRhmopXd8zS4xRay0igOhsEKjW4HyOAISIpRRECsEZi8aYdV1Zyt39aV2id0GgxPWsTG+tJqXOl/vtuAMo0/HcD1vr+nYP03y6vn5+Ot4ZY6Zpcc6PY9gglVzevHmdUkZQpa2IREo/u726zEsqqFvJc+JSmnMtl5ziorX1XRCEeZ5CCEpRP4Ra8/l8qqVp44d+qLWmvFhrUcE8X5x122Dn+bLdbmJcnOus8YScckaSb37r21/6ylf+/E++X2rkmhnAEPz2v/wXzz/4+NNPv2qteXg4TNO9ACvtrvb7aYqlpO1mHMfh8f7+3bt3f/wHv/eXf/L9y/397xCANttx8+LVy0+++a1Pv/L1r33jG96HWguAGKNBaE2JiFprShFzS60prY0xKcWn1wHAWiutzvNUKzvnSinD0AuA9y4EByDD0N/dvSM0Ma1dF6ztm3WtSGNpre3H0VhFOFwu07gxQHj77MXd+zfb7RUzCufQddvtuBnGXMvpdD4cj0M/iOTtZpOKAsS+D9f77d39o64NAICItVLzUlIpt89ebrejUpobGGND8KfTYZpmZhn6HaHOJV8uZ6XIubAdu7iuP/3JD1/tXOduAPDm5qaUCgAsSSkch11M5dNPv/znf/K9WougXpZFAXz3j//4W7/6R1/+8pe1pn4YsFzmZTabYI06nhdEfTmfUZnv/uEf/PY//+/ffvamNdGKQMAaPsTj5TS//fyLH/7RH57+0//xN7/za9baEJxSupSMqJSCaVoRcRjG0+m8rg8vX7703rcm1loRISLrbC56HMM0zwASOkek+r4XkZTWGFcA6YIXEBGZ5wiIBCwgxhjnN3f3j9753e5aa2+MC85Nl9Pnn/1oO95cpsPV9c1msyXUd198kXIex21rebvZ5spzKqTDoNtmM949nHQplZC7Lmy3Oxf6nNftdmOtRcC1rIfjKee8Ljl0g7PueJm0AlsoxXXcbAAgpdwF/6VPvtx3obYSl2ncXrXGIqwUEREAeRu++e1f+Wf/5L9rtYLSwhxT+uJy+Ff/7B9//Vu//KUvfzVlaDo8e3llXFdKCc7VVv7ih3/+l3/2g//4L/7Z4d1b64N2WgSVUq22KtJiYZjP58vp9N/84Hvf+/W/8Te+/u1fsQ6ZARFEBAC4CQBore/vH29uro2xpawhhFIKM3vvnLu+nKec49APWpvQBUI1z+fj6ZBz3e+up+l0tb+NadEa1iXVGruuLyVro6212lhhPhzvAWSz6W+un//pD/74av8cAIdhRFRv3r798Y9/dHVz23lfizTGyzRPcxy3TunQd/p6t9VXVxsAMFoZa7XRxWAtMa7TNE/MYozrwgaI+uBFBM4FBZa19V0Xghduh/NxGDe+86lyTVlqcaE3RsVYldKI5XB6BNBf/8a3v/yVX/rBd78rlhBJFAHRn3zvT//Rf/t//wf/1f96u78lZWLm0/meha0x3//eH/zxf/jXj599ZhlePnsBzAwopFrNEURYUKQ2JlLH05y+/2enh4e7d+9/+dd/48WLD2urRitrrQiINOds1w2tgXO61sLMWplSW4NmiJ23V+q677sQPCIty3K5XGoRZ0PXdaXGUhZFWoRD8K9f3/f9xlqjld6OPTO0CtZoRDodHm6efXhz/WqaTh9++KW+G9+9e/fFF2/GYdQoSApQr8syhN4Zu87vxN1Y554/v9XGKBFOOaYc12XJKRqtC7Nz/WYcW6uAaK0zRi1rwpa5ZmUH33VxnhRURaSVYZE3n/+477rth5/EVIxWRJhzNkb3CIi2lO1HH3/yp9/9Y66FSD0dD+b23d/5j7/xm3/nN16+tMbd372vTb1+/bO/+P7v/fS7f1SWVUS1WpZ1VaREQBmtjdEIiEJaF6Sca2Ux2j7enX/33/y748PjX/8f/tazFy+dHax103Qxxlhr+q5LKRmjtDbLsvS9AWECTaRCsF1H1loAmefpfD6va9TaaKVTrpvN/v7ufQjb2nIIvbN2jYs2Q6ucMi/rGYSNMaEbEPWynLbb8e37z0vNr1+/ub+/d9YMQ2gNSi7O6lxiMGEcrn709mc+BLHBOaUfHu5LyTUX750ANlHBbrzRzmkBlMpau9qaZpVb7YLNy6wUG2tLSkCADACijV9TMSYdTsdaavAvuq5blsV7r5SOMW0346sPP2ThVhk1okhrzVl/Oa//3f/rv7158fLrX/v6fHj/szd3f/Bv/un9Zz/DilOMS65SCyA6Y4CZKqmkgnVGKauV14atnVKpLa8ZQNEP/+QH83T5m3/37/3SN79lrR2GUSnUWvVDYGZmds6t6yos3hnnHRGKIBEppY7Hw7t3b7W2IXRK0dPvR+S0VsxZGI6Hd50qztpaOZd4Pk8xzpsxAOiH8yzcrodxu9mK8M9//jMiI9LGYWNsmDNwmYok64L3fk3RdNva5PHd5+M4aq29MR12uN9v5uXy8HCc5ov3pLAHckTSpM3THKMprQ3dLpfsfS/A1rtlrQhRac3crq72VpPV2jvHDIjUdQGRANBaI6Jub2/2+93bu0eQkhSiUILMlb7/x9/9//w3/xD+y//lX3z/T77/7/71+eG+MaZcphinVDRA512shQubito4owVKUgAo7G3X7TbLmmIpzYdU+Oc//bnCf3X7/PnHX/q0C8jCzNx1HTOnlAHQe19qGsYtgDCDCNdaU4pv377t+2EY+hA6RGRuWqvWpB/geHhAMill53qlCEgva9RaKVJKmday005bt93utDJKK+Z2c/P8dHoEBFR6CBQhvD+cHIBa4rt3X7RSrPXXV69qXbXS/rIkQ01pba0hEOamyB/PZwLV2jpsbp1zS1wIELGbY0OL1Aet1iYkefrpj/7k2ctPP/rwY0BmBkFVGADAe38+X2ptIXTn8+mXvvb1X/8bf/sf/b//nyJSakUkqCBa2ao++7M/+8f/t//L+f5uPh1S5so8rSnVVksVpVrlJtVopxVxa40bIcQUPfoSp9Epv+1OS+ZawZhW6c3nb//od3/31YcfO2eQEQBaa621eZ601kpRjDnGWGtlbrU2Zl6WJefUdaGUUuvpaUwyxlhjiGiZZ0D6+JMvA5AxqtVKigjxRMAihPzy1a3WelmWL968iSnd3jxPabm+eXU6vHPWAerCUtGN2uS4EFYmSCmP263Eqo/HBwDywYOgc85YY4xXmuaHc61NGwt0QVSx1CE4bdyLFy8rE6PW2o19qcqVWYjMZZpAKRCpjUW497fG6Hlez+fTzfVVa/X6+vkvfePb9N//I2jcGhAJNyStXl7vRqvf/PmfF5FYSkypNZ5TKqUqospSGiqlrFZOERD13rVSGrfamnG0LJOr5XrcNTLzmlipxvgnf/zdT776lW9++zu1ZhFhlpLz+XyxzlnjjHbLEpVSiEikEWC7NSH4uEZhJCIWBgCl4tM0pY0WEWOssACCNrbvWCntnI9xyWnpum5dlvfv3y9rNFo7Z+/u31nXuTCUmmNaQbDX9XY7lKwu0wXg6cdup/NBGzsoiI2LQANAETbGDpseQEqulSGucZrPm/3VZrcT0n2/SU0IKnPRBoK/6obtnPhHP/lxbe1rX/7qfjOKlKd/mwhzTiWvhJpbu7p93vfjfD40AQMAwoN3+3GY5nlaIwhOOeaU11w0gnoaY4lIKRGJOYpxhAJLdNZoo1CAazPO5JQRD37YXg0hgkqlzdPy+//23263u6ubWwBUSmntQwfO2if8TkSIkJmJNDcOnY3rEtdYW7vZ74QxP/V3ZKUVMLdWD48PpWYiMsaJtBB6ranv+q7zpdTXb17HNXkfgrdGm1cvP5qXy37/alkerVUi9MX7u5ur/eV8Dt3W2dzKUkq0utPbMSBYZm6taq0QKaWU7hdubRg3StnZppubvbO+650wCGKrEJdz8IrKWoVU2MV4uEzTs6v9duz73mkdci6Pj2el6OpqZ7RWpJYl31xffeOb3/z9//jvDZAGcka9utrGdZnnJbMIQow5tYYASKi1MUqXkueUEDAo5RSIgjkmEbRGa9JVFLMQCyVhmboBgvebwU9Nvf38i9c//en++hmRcs4krJhxXuar6z0hIkJrDZBAhBQiIgAoZVlYa8XMnCoRKKXXuBhjrHNaG10cIuWSz+fL5bI4Z7qud8G/fff+eDzeXD9TWqW0IsC42S7LtMQLaa+wGqU2nZ2mEwgao8m5x/uZufRDoIeH9/N8AVQ5N0REIuZ2Ph1SSqfToXGNae37npS+XCYALrU5kpKXh/dvm9jEVgA6b7/80Ydf+/KXd9vBGFVKfbh/KKVuNvurqyutFSAi8bOb61/+5V8x1hCh1urD67235jDNa6nAnGKMOUmr3mjvehbMzEWk1FZrqwyxVmsdEq4pLbmsrQICiBJBBhDBUuv0+MBx2gVNAH/5p9///LOfny8XJGW0Gvv+vKTH06K1bizMQKhSSq3VJ4AIgI0xWutahVABNKXBGEUk3lul0ToSIm3Mbn/tXceslmV5//b98figjR43w83NbT9uG8PhcB/6bVxnY7paJeV4fX0bQudC6IJFEKWddX7ogz6fj133wbQuOz0AIBEhmNvbl0S4xkW4Gq1a5WVZYlz7rieQebp417WGP7lbW43f+NRuN93VbtMPgwjP8zxNEyljjWVuWjnWuaREIAj44UefXN/c3r19+3w7brvu/eMxlYIATzyJADDLWlm1BADGGq2MscopIqQ1pRFl029Ol1Phhk0tcemHHQIoQgbQqBqpaToPhJvx+nT3/v3rn+9vbrhVEUaUm9021Zpy4caNueWktdJaa60VKa0NIKYUc8rWubSmGBfrRq3p6YiIyP3h4I2xiokkdIMAwJL2e7eucVoioHa+D91GHVRlfTwe4noy2q5LUkoDVmsti2KpwzCmuPR9rzdd74xOSyLSWmsAQETrQhOwlp3z3ve1lpRmFqy1KUUpnsftNZZuenjcBuz7zjkLAI3r8XjIqXgXkBRAK6WY0DnflZKNIufcJ5986evf/Bas86vdfo3lkrICtlrX1gqLRg2KmXnlaq3zpJkzgdJKKVKt5sfHh77fIilNSrhVROIKwEXYGg2ExloUvpxOlbkbxzSddttRa9NabdDG0Xe1LfOslTHWELLWpjVmZlJkrWPhUjJLtbYjGrnVx0vsHJV4yrnsr18Ga0+X8822s8aI1DWuImisVmrUPqy5TMezVjWEAVkNw/7u7rOvfPkb83QWSUrROk9zRkLaj4PS2PW9no+vncrXz76KSNpoa03J/PD4cFnLNqguhNpqKcX7bpmn8/EujLth2IYQRPJXP7jejp4IW2ullPP51Jp0YWCBlMUoePqIfd8p45SpWHMI4Ztf+3r+/GeXZVkrMwsS1saxSGosUhWRIQJAAKjcaqmqiSEi57wbLtPlHJM1xnDd9cE5o5RmUqAUCOXSDIomgsrT44FLuRzuuVVEeXp+c85KGWOMc84YA2BLqQCNmYkIEBAJ8YnMqsYQMxTRlRHQgOQYo7f2VFOOgmK1ssH18zLHdZlPj/3mptveSOimZZ4fjkXwerefp0NKqzYWod3fvXXOG7tBwq4P192eW9NXzz8dhhGpArCwGKPjuhLSi+v92HeVs1KmVlAKnbOoSAS0Ca1x39vttvPepZSOhwOLWOutocPjXQXs/AgsIq3UCoBEFikSMiKWdVljPC5rFQAQJViEGVErnUvRSudaGaCUUmrttKmNK4tqVSGY4LlJZ+y0zqlY67tmbAPUiIAGjWVOopS2AVuJa/zZn/7pz3/0w699568Js4is62pt67oOEVOKShEiMQsgEhAiKSKtsIl6f7o82wRgMNo0EGP6jlBr/cX7t0pr70M6PzAWs/+oNUbUx8eHGKNx3lmLTSXyc4Yltn5z+3h4/+L5ByE4FhRmUgaJNtsBBT9/f09+vEYbrHUsyMxPxF3f9VphY+670RiV0wqojfVKm+BdCK7rfNcFpXBZl4eHe2bZbvbO+VJLazmn2rhIq/dffCat1VpZoNYmwq9//qM/+8Pfe5yWwsKNc22l1lg4l1JbJVKVmRShACE4rQRBEQBCa7W2SkSEQMidsyyw5pxyYRYAyWXNtQiSds73PvS9NnY5z//xn/+T+/fvSRsigtbidBEGZgYQACACa51SBomUUq21J8pPQSOlSNtdrwavuKzLuq6xlNKMc2HYeN+J6nMuIfi+GwS1oLLGceNaV6PSqxs/dr7WCgK1pi50r16+Sjm9/eJnwFWhuru/u79//4Tk3cHu6u7d6+34DaV0bZkaDH6jrW1A3JrRmpRx3jlnvfdaKwDOuczznHMKYQyhKyWKSBd6AFyWhQi5pvPj47i/DWEUFufCmzef/c4//8fv375rgLHVlLNRCgEZ8InG0kY15qANQdVKeW1QKa0UAwoqILSanA+IqJWtrSmBUopDBG2FWUph0FwqEGhtkRQivPvJT/74t//V3/rP/mc5p9D163xurQECESKACJwuly2is6q1CiBKq+BAYw8gSBg0lVKjAkCVUrq52jhjjDawuy6XyVqntOIGLz/4KJcE0FJcEVAROqvXlLwBN+zPp/O6LF3fXe33N9e379598dOf/WSaJkVKC6c3P/kz/uDL8zQxg1IKAUEAERGwtRpT7oetc84YbZ1j5pTiskylNGvdMGyVskvMJSWtkDXFVJ2zRII6dJtNjNGHJMKK9M9+9Bc//rM/n3MtjVtrjdkQEmpu0pgBSTEjtzU1JKU0VZFeKa0MKyW1acSvfPD8ert/OJ/ePV5aayBAiAgCrXbeD/3QdV5xq9KMMUppH6xwu/vxX/7wj3//xZe/ut29MNa1JvM8dV0gp2rJXLLWJMwIaJ1jkdaqgIg0rTQpvHvzVmvTb/Y1Z6XoeH+PnN0wdv3onGmCx8dDYx7GXUmLUVqUarVoY5HnFNdx042b54+Pj/f3D5tNv9vt37374uHhXpFCBA2gtAvGuWf9h6VmrRW3FpvUdri5vtLGbDajtZYIEamUcrmc5nkyxo/j1hhVKsbcEESYY15cEGc0kAKpxpiXH3+ZG5RSRPiyXv7iu9+9P5xi4yoiAIaUUtSYc2MBMIoAUJEGgCqcaymttca9AxvUJ6+eaWnPr/a77W67G7ru9Ob+SIhWqd0mjMEoEMIEMQopaIXFxVqBe+sc1PgXv/fvl2WuKY3bXWuitCo51pqd60kZQhAWUoSI3ORJC6I1EakU48PjBaQ8e6aVwsvj/fn1j5dx9/HXvoWkHo8nIj3Nl1r55masrSCZFNeu74hUTotAG4aBW91sh5DHZZnevHndD+HD7ZeWZV3ipGtNX/rqN7Rxl8uJG9uu01o1lu1247xXiozRIlJrSSnN80UpNw43xmjhllOt4taYxkB23JwvCICKJNUqnHOOT+s+CGvlP/vZD/7yz/8siyAq5CoiDFKbEIJWSiEpIkHIrWmlQAQBDSkBGZz+6Pn+049fGGWmy+VwfARQz66vfPCt5CdEYr5clNLLPIuA81ZaY8SY23bTfN8ZpXKd37z+p3/++3/w6tNP/TB++OlXhu3oXEiwlrQcjzgMA7c2p4rI47BR5heD0zytXbfhFpWSoR+nx3uybnf70lr7/v39NK03N9eh2yqtlZLaBECIyDtfco0paUWIGGPMuabSxs2m5BjTVOq82e591+kUox46EAmhb8yI6IMzNuz3OxGIcTmfVxEwRlsb+q7n9sT5cSkZQHKJvfcCoBSQdiVHa3Wri9E6Z17XdD4f+y441z778Q/v7x8RFIEoxNq4tIqgvXOaqDYWAGZWiGtKVmsE9JpeXu2//eUPdrv9Ok1Lg9ZqikV1w8rnVorUKLVVIRZYLrMiVUsRgtCPx8sU3CCoY6yJKgu0Uktp58ej0fpH3/v+B1/58mZ/PV/OZT6PVzfPPvywHzfGd8F3T4qQJwnp6Xxx1vTddW0ZSY37G9v1w3bkxkTknAOpm8GzYGut1ZzL0veD8+Hu/gFRWUtPfzkMIy3ROuudVgov0+l8uRjt9fPnzxs3Y4z3rtbKzMaY4N35fIjrElPSxu13e2OctZabfnx4MA6d646nqZSslAYoXdcL11bWWqsPxjtXK2tjEWlWS2nEKb774vW0rqINAoEwAgpLJa6tVW4sYJQCRGH2xrE05tb7/mufPHcE9+/eT1NkJOM9N+E5ZipQi7TsjAFh0sY6bMzQJFV2ZMWEtTWJSQQ1iXMeAIGwpEJKP7x7u5wPpUqOq5U8juPr2+effuc7zz/+tO/C8XQYhsEY01oz1qxr3WyCUn6N0YduGEZGPiyQaVRmWuZlHHzwQ2XUDHF5bC1pbZ4YBe/0+7v7nEsIJgR3PNxZa7g178aU0/3De22tbkwAsCxzinFCiXGNca21ej9c7699cE+HkUgpjaRAa4WEznUgahgCkkopa6UQZegDIfow3D/cBe9d6OJQj1O0UufziYiqgICQUpyLIAhzaZVZnLVG68bcBBBBoSLCj57dBIS7+/NxmmNp5PxQqyFsiouyaZ5Kq/vdPli7rgtp20SU0k2klDL4sKRlLUUBIkKiKgC8RARgQq10XFJtfD4eLNd8vuTLOU/HdDkqo12/1cogojVmu9mGDgCa1lRqQ1KxZGPsWrImNsauOaUqbY1IKudCSmttEZEQALjWOk2XYdy2mmO00zRvt2Mt6e7+9atXHz97/pH+4u3rlDJz01oLSwh96DZaaWutUlqk1poAda3VB+esN8YIS2sMAO/v7s4Xd3t7ywytNhQpaYnMod+OQ8/MS4qXdQXO5/Pp4e49A4gIEnEDARQBQGARFm6ttVZLY0F0hDeb8aOb/c7Zt+8fixCRYZQ0z4ZdQ2UG54zB0HNcH0/nzgcgiqeT0cZo3bilmMhwqw0EYk02+HWN0CoEX1vrFXJVgIoBkfkyXarGVtJymdbTqeb69d/8LRwGay0IswgRChBz6TonjEtVWPPtoBTZeY6CFFMl1UqMtXHXdcbYUnIpVZi7wX/w4kZbnwrfndo4brQiqwOBHA531zfPdErZmuCc9943Ju0cSE05Kk2AjWspuVm3KbkwP9EUSgS0MlpTjfPd5WyMGvqBSAmi1n65nJuchmFDRNO8IrMmdT48XB4enhpva1UaCPMvgAhARaq1lgqygAB/aXf1nU9eOqfPU36cs7ZmjXmKafA25tz1Y65N1oVJMyEz5FalYSoVAJTWgFRrBaQ1JmNtyrk6n2pupVYErk0TaOvWXIkbtzZNs+5C1QytnQ7nn/7h7wrCr/7d/8zePgPAvu9aa8zcGiFA4daEUqvbDhqLNfTs5poIU4wPy2S07/o+dN26xlqLNtT3PQArpdcUkWMIA4PuHL/64KO7+7uUon718hNArCUD8Hl6zCdwigDq2D9nFm1srS2mFUBSLNZqpXQpjQidcy9evTidzoSIyMaa1ijG3BiI4Xg8GmOmy6H3/VqVMEhNIiL4xIQAKYKCjUUACEAbQ4Qt15ux/+Rq12mYc01NjHWx1MQSum5al6CVq6wNpVwYayoFRCpUa2zoOxRurRoXElcjrBTW2rjK+XJxPjTkWGtaVi7ZWitK15wdyNCPioBrVcNQ0ZzP00+//z0h+83/wW8+e/nC2s4Y86T5BUEk3iLnIu2JfVUgUuc51lKDc6SdJtBKp3gEEKMJlbJhlFq81TdbzyJLSq2U81TmtQKctNYq5SRQnfW1FoUYwka4KqWexuLCGoX7zs/rMoxBaTvP53VdldI3ty8BAEmRslprIkREY0gElyVezpd37764uX6m7Xh8fB9Tqk3IEBCiAqkgAAj4pOJ/oiKM0x9fbfa7cW3ycJwOsYAIkcqtJRanzbxeRLlXm31qiVs1JggXBmkAIKC0bbVSKQKIyEabVBsgNG6AYJReakatYq25NmvM07RjnYunRwp2Op9VGAE1nC5/+Ye/R8j6b/6dcbtHRKVUShEAt9ud0a05nVJ8En4hwJpEEfquX5alNsPcckogEkK3rFUbl/N0Ph23m+3740pKE9K0pLUQwapZmiJKKSlSxlhvjFHUjTvvfck1VyHgxvnucdWamMVax8y1lXmZck655O12b50Xqa211vLldHBh6PuemW9vXtWW23qRWhE0EtRWgQlINWbm1pCYGxKV0jTofdftxh4A70/zm8fDMZXBOSQdS9GKOt/lJvPluN2MKFhqJUSFihEaErQmOTtrDalaCwtp40qLf9UNm9SyCcOyLECKWztNZyXc+6uc85xTCJ60B2ZibGs8zfOf/od/b8Pwa7/1nxBRTvmJvj+dTtYaEVBKK41U0RhtXX8+H6111oauD6WUXAoRdc7N65pqqk1C6EpNY2e11gjw6nbzwYvbmiOdzwet1Xa7d97f7K+sNYjovWutNm7cavB6M47ny9kbU0p+ckCAiDFmGIba2vlyJqJSai0t5zqMW+vs4fgupsVYKyLn6SDYSKMiQkSjNCAKICIBAhKCACAC88vej9Y+XOa3p/lxzYWFSFVuDQARp3WaUpljvsyL1NxqKTnVVhWCEjHWamettQCA3LRShmg79CGElHJeFxZZ85rjQoSIUEplbjWtIhILp1KtIgTJOeUma2rn4/kv/+D3f/iHv59T0tZ73w/9CIg55zXOh/vP8uULrZUxJgQzDL21mgERaV3XVtlYw0S5lJrj8XgutTpjO2+cBm4FkbtgkUD3/Xj3cD/03Xa7JaKcJ+MGAW61INjp8rgc31xf395c7YhUrYIoRCgizgVj6ObmWYx5nmbn7OF8t9nuvA8pLbU2pQw3LnHFVpTWAgDCCrUglpxbK0BIAIZUqY0Bng/jl55fxZTfPZ7nwrFx52xuTZAskSKVckq1mtDHHBFAyCitQSntbMtVGEgp1CaXQoQoLc8TM4duiMZwq0qMs7aZ3FpbYyolgbRVYQWamceUzscHF0bb99qoeZnqwo9v33//3/3LFNePvvmtrh+90YrAmCASoOZ1nSKfnpwH3vvW2jKvtZacCykCkdPhvKxZaeq6Til1WRJpl3PqvFG6my/n6XLStbTz6VTi4pxjZgBkrrVWRMol3b99Deu7Z9dXNy+enS/rsuTttuuHYV1ia7WUxq0561OKXRf6vjPWllJKKV0YWqvn+bysizU6eGesAUABYWEAFEEWoSeqC1kAvKZ5WR/XvDbOtQkLNxFDikiEEcD3m53rBWRaotaODNbG3lBhtN6XlKDWda7MTSMKsyDW1lROzhhRupTMi2hra61NWsq5SAMRUBoFDutCmpxvtXEp7H33cDw559Td/d3Pf7p78cqHodbaSrRbMy9rN161fk+KpsvldDqFELwP4zgAQs4p5yhiOu9vrvfrmktrIPxnf/n5q1cfOGtq5fM8cTppEJovx0E3BEgpK1KDd0okpfJU8ppUY611IeUEwohSa/M+iAihSnEGzoANAGNMXT8QonB7WiPW9bIsc2NorYVhM3QdAQoityYiiCTMAE84NDhFV97OMZ3XnGpZU2KU++l8mqaYc21cmjBQ1w9ca2Uutc1rbCCpFiIEhFQyM5SapZW4rtMaY4zMXGrWRgtRQcy1Ka2A0BBprZZa7y+XaVmLYG4kaAW0IZXiSoCNZZrm+TLfv35zvn+rFYmwNjrlnHPMKee0EEjfd10Xcs5379/Ny6XWQkTb7XYzbkkpFhQQRFVKU9rVOPUWiKTzpJRJtVDoei7L0PdEirkRVE2ilCbSpRQGMEq1WoVBEYSgn7CKpzKqdCDtlFJKk1IEgCWnlJaYl2k6LUsEgD4Y67xyg7a2cpXGIIKIWiuAp++fmUUrUgpzbbnxlEtpNZWikIgo58QsBcn6gIgppVILKyJtW5PQDSCQchFBbTULVobD6cg1z3FdciolI0sp2WkdjLbabDZbNK7UpokqwJzWtaRCdKocSSERIi45ISkgteYyTfPd55+t0znmXBqsMfddV0Vp29fKJadx3BijrPWIJsbovdvtdvura2M8EpGyAhahfvuXPvTWPGHB3rtxd9tvnlOlLuw/JqWMMct8Ft1rP6Q0vf/ip8eHL/p+QNvPmXMpLKy1zakqRbW1lBZCLLlpsptxEzrfWm1cUArXejqfp2lSiJvN6EM/XWYQcVopRYDEAAjwNHoCogiXyimVKZfETEBP69449M4op3Qq1VkHiDGlXLIgFRYByDnXUqCxQkQQrgW4lZQayJwzIrTGMSVp9UlkCKiMsUjaGL0fN4YUAafaTtN0WefLfIm5TnFtIoRYRZ70E2ldDl98cff2C0JM86yVRuMjq8JESFqbEILIk1IAx83O+26azufzoe+7rhussSBFab3ZbK9v9iH0uXJKuZYiIIRkh+3VvK4ggHZ3qkNha7UDoaur5x9/9JXbj75R0ac1rWuupbKIADhrUozLcjbWamuMsSml4/E+rYt3PljLrZQSh2FA5R8Ox3lZnfOKqImwiDCX1gDgifkhUiKsCdZSU05aUWUmBBQpKcWSi7RcyzTPp/MZiIz387q2xkPfP3lOoVaj1LpGqVVaccbkdZFWNUoqdY4LNFYC3mhF5Ix+dvNst79BUr3vUKnU2rSuIK3VCqhL4VqK926OS6mptRqnS4kLlMRlMlYXJgAkaCmtIPJXdL+2VoGwNX4Y9qWUab7EdbFObzaBlMppGXtvNXKpOWdu0SjQUi4/e/PF+/fvjPGFgcZ9cHSZL+clmlCC8DSdc6rC0vUdC9d1NQZ9CPM0e+/3V/t1jXeHh/P55Jy1LiwxCdc++O3mE+u6z794j6A2u70fRkSQxu0J1yMqgCzCzCLijBFhDWIIn3qA1qq2ZknlWr3zRBRzWVN0zlZBYI5x2W831BqDVK5G6SYyrVPOBQlzWjPXXRgIKK0JW1XGW0VlXTe3N6KMS+Xm5nlZL7nxtEZurIFIyhrXwfnaMrfShQFQl1JPh8cvfvrTl1/68nj9CoSxJUfkDRUB42xKqeSCENUYToe7eZ43u5v97jrlfDjcO++89cH3ILDGOF9OSlsF9XQ8aWvoez/409/5w+/v99daq91oR51qnA7HS6k8XS5N4Pzw/vj2L6wGZ32tVQByKtpYpXTXdefz6c2bnx8O98H5zrmclmWZaymbzcaF4Xg6tpIF4Hg8SK1OqSfFJyISETzJqQEAIDdOpTFIFT7NS2sNAZRSTYBFvHHOulRrqrm2xsyAWGqa5rMPwWqjSJUYCRGV1tqsT9pewDmttUZNSEqltKZ1RpQWo1a0u7p6/vLV9fXtfrNBpQCRlfLdiMrWWoXFGKuMIaVFoAhN5/PlfDS+K6Wty9lpKI1TLt67Wou1zlhbaj0+3B3ev76cD6SUAO2unqdY7h/eI8K42QNa0h2gQSFrLDelT8fjzc3ts9sbZQaiFtcHIn2922tjvbPG6v2zV2W7HXd7az2iAHgAEIZS07u3b3Nr3rjNuOOaapyU0n3fQyvaD8fj+XJZ397PRrEmRBSNgICIJALMzMJICvAXpagwWFJckzRm5lpba9WQagzO2dJyLqkxzMuaYnp2c7Ppt1qbyzxtur5xnWtypAxIBUkpIYhSJASp1pxjX3tAKKTycrHWYHLj/togKOZbkL94925eFtDWd/1aKgprINRmLpUJAZEEgFQ3bBVhaw0RhaUJG2MRVSlFQES45iJk/eZmGHcijUicC7Ddn04wzZdUSvC9cd393Vvm6rtunqP+5a9/1YZhjRFVddZ0QV/mddyMAIgI3nv7/CWRAmRCLKWs6zJNkzYaAKxzvfVdGFjydFpzY6vBO8fgUpaUy/tzefOYnm/w4f6xVNZKiWQU4NaeJOCaCAEQpEk7l7y3mlsDkMYMICiw5gzACLLmtqa0LJMxBo2V1ozWIXSn0xFEOmMQ6HQ59cEToSAKS6rVKJVrKyk6bbSx3Jom4hwNXRmlWvCj7AXharObUp5yySCkEBqi0toYyVUASq3G2nEcCPjx/j3ZoTS5rHE3DkqDCKxrrKV0ne9C9/zlS6MdkrRaQSCnmHL0wQn7eZlKTv2wC6E7HmZFjCj6xYsPSeE0zc6Ic0YEAjdjzNOEohQ5Z2qt07SklE6nU4yr1ub29tnV7jblzFxynoVbLhmVdmE01kxzOhweLtMq3ILFXNKf/vlPfWaliID/ah0TAWFmYVZK1cbHNXdEVinEJiKl8RMK7I0ttZbK5+nCrVljCGla5t1un1MSkGm+NK258Zxyq5VQaaLGkmIsqFpriBBz2hprjc5NpLY+51aqtz4KmFKudlc/efduWua4rgYJtW7C2mghXFPywygAxlmlLZF6OM4/fH3/8srfXm2JsJQa10hEWuvLkoPvFOHx+OCt1dpOy9wab7ebaVr6brOs0+l471y321/nnFgWvcal7/y4GRXCE/7p/UYpBQCIUkpdluV8Ok6XcxPSyrx4/mEIobbKwsxtnqdWklIKAIfNXinKueVcjLGbUX0gLVBNma6vd1Emay3IjIAKiRDULzgZQhBAKCyV2Wn1hI0iQCpFIwAKCzxppEmk1QLakKIco9VGEa2xrOuyDwEZUklFmAWrcM2VRbw2tRRCtCpbUoSq63vmJiAMWLkGozrnCWnNiXOxoSNFYGxKWQO22lCac93uxYfj7hakxPJ43dG2DyICSLXkViMpdTxcUhOCUWsljEjEUrfbbWvcd0EY53ndjLvz+Xg8PoQQtrtn5+MP9XY7am2UVvSkVUKoNcfY1nUhhJzzvMY+hJubm9oAUff9ACAKMOdY8no+HxVgP2w22y2RSjnmXGpNAOS86op5H4/ehWApauPHjX441gZPDz6DNGFmVgqBOTVeqt5aVACEgMCtVmeMQsq1XuYpp+iMAhBtVO9cF3yrpdZmtJmX+dRaq23NUUSUsSgCRDFlACCAxpxqioWQVEsJByYR5qaUFkg774O1TKoiCkir1VpnrF5TBgCuxQU/bnalrCA4dN13fmkHKNMSr/abZT4LV+u709p6b2JcQjf4EFiklQKUgu9FgAjXuAZvjdWbcX86H5SZfLfVztlaas2FueWcay1rjLVWpdQ4bLzv+mEfQiBsb9+9Y46I0PWD0nqezopAERGp0PVK6dPpqLW11lnrH4+np4m/1Gxc6PtQtrt69Fbh0pi5MTOI0BPbjIqQiGDK7VnnDSUQQaL/H4OmWlvXhaVp7ZRSJZcmEEtFAKv1NEeFdJxnQlprI2nQmBRxa8KSSgvGlFbXlXujVQciXEsurfjOI8rUmEicUbFhZQERQGyNlXGIxEoVZtuFfghaqePxGIJvXAAAyGqlUkqAGPoOrW6tprx0pJy2j493AIJECJEQcqmH46ENAUS00TfXz97f3xul9Ol4v64RERCJSBlrx2Fnnev7vjUB4NaYuboQjNa1MktrLcMT3Zqq0tYYa4x/PNwjoIi0lknZPvh1mWquH3/pa5fLtB3Hy/k4l2ZICTcRBkBAFIAm8iTeF5FU29rEG60LC4sANAaFKpeyxlUhISIh/sLmR6BQ51a99zVFAJxz5NYaPHEwjVsDAK3UmpPTSimYU9qKWGeBEIXLGhFRAThFGqCUPM/TjXPkrbaaawUEBJFat1fXoQshdJfzqda82dwsy9R711pbl1Vp651DEmbnnEMEpdQ4budlRrQlt5ymJZYuhBhjTHl/te28tdpMy6QVuZubHRIBgNaqtUakEbBWzqV03ue8xJhCCH3fz9McnF3ms1YY00qovAvW+ZSiCITQpRxbaqksKca0nN989pPd7Uc5V2aw1mdwpDXwCvIL2ZOIIAICPFECmfkQiyWlECszPcHK0GJupValtEISFtIEIq22zXZcYiw5CWHh+vS2tTVFKpciCMhilarMDlQTaZUv09wNG9eT1rYAQm3MDUV6799O85ojETqlRaCJaObGrBXtnz0btlek1P7quuQswkoZYxQApFy2m70xbr48LPMSwe+2o5LsvU9VmqBCIG3X9aS1VcqM23EzDuu6aKM3444qi4ACUeualjnf39+JNCSaprmUAgiIapqmaZpD15NCEJkuU1zi0/lyPlhjp3kiUsfTMca1tYoAd/d3l3lSWrdWK3Nt1WjdbbfaGEUgIk9YkPz/0QgUltbanLIhturJKIaaiFt7chE9zU0osKa8xng4nQ7n86brubVcmtVGBBr/or8QoTAzArMYIgRoInPJUbiI2H4kTcwlLXPlhlpbYxs3Z42ylojQ6FSy0hoBxu3m069/q+vHVqs2pu97AVGKAKTWorVSmoBQGZ9L0pi9oWVdaqvyVGuBcm7ed1pbpc2zZzfDMC7LwtxYMp3Ppy+++HyeLufz6fM3Pzfa/pUQA1iYG7da97sb87T6hlBKMlr5rg/dqI3t+y6m1ZinFKezMaa0Ok9TSfF0Pr/44GNnrdHKe6dIDV0/7q83XSAA4SrMjaUyl9ae9mOjdG4Qm2ydJhCFolBak1wZEBQhIjaRVMppmpZ1vUzT4XTsnFeKAACJRACRnhbpp/ojwCJSWmNhq41ClAasLJNuqdQYc0xIZBTVUhWA0xpRxXkuOQkpbfSzD19urvfLMpWcCYEUWWNE2hME9KQCQoDQdR996atXu1Ek+84zMyEDcMoVUAXfibQYL5pkulwul/M8T0M/au9CbaW2GnxotbTW4lpKbiistBeQfugAsNVSci0lpRTHYWTUXOt2uxXh1goADH1AaM76GHNcU9/1h+NjTEWRctYY41jAGOx3V/tlOl6WVIuIsDAC1MZGkdXURIRlqryxanSqMlgiZi61goDTCkUaCBIpIqNUeHL9ktoOY8oJ4RcmFwb+RVUTeIKXU61OOUPQSlHGLPPCWqeSUlwdAWjTWauJFCpSIgglp5YzA3TOfPmXf3UYtykutXIuyTlnrXfOG6PneWm1IkLOpVZe11mYd7t9LoWZiRSSjpdTLQkRlYLOO+H2/v3blFatQxNDSOiM88G3VpYlLesioI6PD63V3KQ1AcCaY4przhFEtLFLyufLxVhNio7Ho/cdIiLBMG4ARSk01mpjXzx/VXIDeWotxXu32+xEW9cP+94bpTUhCORaEZ+8IUCILLDmtlYOzlmjiYgIaquKMFintSEkrdBa7bzzzgfntMagde89Ij1pgQlQRAyR00SIClCRctp4pW62m95ZRXi4e//Fz352OtyXlBHV4L3VutZCSEggzFwy5+X6dv/pt77tQ5DG1jildK11WaZSkog88ZrLMse4huCtUU/cy7IsOUci8M60lpyzWmvv3fXts1JbYwZAa+nh4b3msrphz1xijCy8392KCKBYa7a7IadUSm61llpybk6TUirNqevC9fXt5XwSwVLKZrO9XM6IgojW2s1ms8SSS3n77vPdfu9dd7W/RoTD44Prhu3wlbouxyWWVlvl0tpTeScgEEEQFilNekdWREAaizxtDMAEpI02SNzkMs1Om84Y4QrQBufKMAi3J3iVGRRgEzCaKotws0ZvxnG32djgBVsfnH/24nJ8LClpY7rQeR8aAj0ZIluuLXXWfPs3fuPlx19qtczTZHzb7XZaGxbOKcWYQGQcN4BApJdlRuG+Dyw4jpuS0xrXuKQudMaaWhpg09oeHg8gpJTyvvNIep4vxg3Om93u6vr6mXJhnnLfDT4EQM55SWmNMYtA13U29FSzMXEcNiI1phUAL5fTOG66fjg8PjjniYgU1soPhwMigECMURsLIsbaly9eHU+PE2Nn9VoMS27CIsAARoEA1CaakEWWlAdjmzRpIiIESNpYRbmJsR6UUYoOy7zGdec9cmUQpXDrw1pLbQ0bPlmXDOnciiJaU35zPDxO09Wad7cvxs3OaG2inx7vhiFQ6HOrT75Ybq3G1QF/41d+5Wvf+Wsh9NPliATOKgRYlhVArDFK664f4rI+MaytVQAFpAiIiLRxl/tHa6jvxpxXZuw6JyLLGgFl6Lc/fb86h3rYbHOandv3fV9rW+cLM2tnhHm6nJf5UktZU+77ARAE0YWeD0cBORyOAE+YlZunOXR91/UxLtY4ZkYou82G+742nh/fsohz3TiOX7x5/dOf/zi3tr+6SXzHzGvJuVajLQAAIBJarRTiXFpn2REmraxxlaWUqtFwq3Ncm0RQRICWC3QBhZ01jIpINMKTy6xgexqiaqtW0WlZqviHNP3o7vGjL60fvfp4DMEax2SMVjXFnBIAWuvyMlHLH37po9/6L/4Xu+cvlUJmZhFFujYptQ2da7WUWrqu08os67KuMxEOw6iUEsbGVYC3m8F7JyKNDQFobdYlirCzNouZ4nnjDe2vbhC51vrw+FBKcdZshs4601pJ8wQsSuv9/qYLIyCJkCJrtBau67pobTSJcJuX+Xi470KvlIophRDGoR/HwTo7dB0vD5wXpXXohs/fvDGkP/jgI7u76rquM8Zrw/ILol4hqif5NKnauDEq7Z9ChWstmgABtDYiYAlvNpvRW2Ge47KkPC2x1aKUcvrJFQGDNcFaAX4S7TXhj199uB03DKK0r2STELmu77qFsZRitam1ImlI8Wo3/O3/4h/sP/iS8d0TYE6kSGsWuUyT9c6FToRqbcs6A4D3AYDu7t4/Pj7+oqIihuBB0BiHSNY7pcyyrt55a50If/OTZzfboPu+T2sn/OTjbF03lBIVcpEW11kZa92ISIjY+b612rheXV/XWkPoGkutGbldpsNlWozx43D19t1rROz7ToBzKoDqxZd/+XS+aOeNMR9++GHn/N37d6W10PcxpSycay2tmYbGKK1Jk3oK01hy6Q22lltjrbUmksZTyoN3z5/dvnj50f3d3U/OJ87itWYUKBS01YoQVWUhQqttSrkJKEWCWGNcU3y+vfq1r3/LbbaHw8N5WQiwxNg5Pw5DK7W1VNP0S7/269/+zd8i0qHrQRAAtfEi4py+vdk9HQjnbCmr05SZc0pGO3bdPF9ijLvdVdeFWvl8uYSuI9Jd19eaidSyLtr5FzcbRfT4cK9FYBi3d+/vREhpLDWnFGtJOacm4F1nXch5MdY55y6Xo9aDUvgUiicMKfPxeCBlkezheNZa7XbX9/fvvLfWWm1Uztn22wGNdfbhJ3/58sVzAFpTjTkXQeVDj1qWlaExADBoIquoSHtanVpjUgQgBGC0sVonkT4Mpht/9PlPg+2G7dXdu9fNWq81IVdugVAZ2xhjbV6LIjIgLMKNIzcB/J///X/wwde+HueLlfqHf/QXwdmPnj8XBGNMF5xCDNZ89df/hnMBgUEYUSGRMUZESsnOOU3qEtN5mvaD15pU45YSoAxDsMZ+8fZ1a5XoeUo5xguidN2gjb5cTsws0IIPCFJr2oxbKqVo7aZ1zjm1KqfTwVgV18UoE/oNacPc+m4AdNNau64Xkfv7uxhTiiXG9MMf/TDm5EIYNxsiOp1P1tq+65ZlAoCrqyutqbWy3+9LKaXkrt+GMCCFw/E4xYhEcy1LrYIqi9S/oovhScAlkmr7BWcG8DQTEeF2u13WaT6fj8u6398wYXky9wBopYAIhS1hZ7RC0qSQpZbCCPen01/7zl9ftPs//1//4f/pv/4/vn1/Z5SucQnBG2sYBERA+Pbjjz795b9eS26lcGvMjVn+Ki4CWpNcam6SS40x1ibSWom51saNnTfPnr2wxj883K/r5KyLy5Ly/LTUcpPNdgOSYryA4Nu3r+l8nljkan9Ta3p4eDddjvO8aOOU9esa12U+Hg9PBv7WOITuKWtK5AnB4dvrm+cvP2KBcRitIUR1Pp+dC0q5ZZmstU/OA2Ze1/lqf+utqS2fLmcr+cNXH1RUMRdAJKVSaUstgqDpFwbKyi03VoRPdlSF0LjllEYfPvn4UxPG6TIprYLrCIkFBMAo3fnOaI0gTpEiDNYCKQFS2j7b3XztG1/TN6/+q//d//5v/4/+8699/OlXvvzVfhiU1rVUqdVrZK5f+42/dXVzbYzNDVARInBrKeV1uiyXszACgCEYDFirlTLKeduFmpbTZX44R0EYxj74vpRqjBm3+1brdLmktZCiEELJpbWWYr67PxIIzvO82+2GoTMkCJBzQW2WZTncv52nqeQVQLgtu9HVWs/ns9bOmMDKkdYffvjhZrO9ublBpOPx/ou3b16/eZNS3my2T1+6D945e74cjDHG2pJXEUYuL25vtDGltufXtx+/fDH2HShaK08pEypEVIQomJsQolbqKfbZKmUIWmvX26uPP3h5u9uSdVpr46zI0xQFjZtGAiSjSAHvvNOKAODD2+e/9o3v3Lz64Gd/+ZPv/eEPv/61X/rGr/5qPB289URU49JKCcaEvvvqr/4aMItIzMVaX0tlbkSolDnevU1xQcRg0VnS2j7hPdaam5t913Vv7+8aN2uVtWq/v6nMyzKJ0OH4OF3O1oDT2mhVCtwfTq8++IBC54EL57gZxpvdbrvZsgg3IRQfgrWu8/54OqSU1jWez6ecc83ls89+EuPaBYeIhORdsM6M26vgwm67uyxLKbXvtyklFMwpIWAXNkRqjjVmfvXi+uUnX2GAIfiPP/pov99bY7oQSNlzLE+aNa+0JapNUIAIRLgxE2Fn3dvHu3w69MG8fL79+Wc/dsb1ocNfFB9QIgzgnfHGWeN2wQVjSdG3v/Gtb/zqr/34i8unL9ytfqOUjo3f3b3pt2MpeU1rzElpvb2+3l3f5FwQpQuOEC/TlEtVSg3bvem3McXj8XFdpuNxWte4rmcRts5p13lvg7Mx5ZILkQCwItsaP1V/htpaq9xcsKQUA/lgdY6XvF7SqhAgWFsq19I0oe+2gjqnVZmhFg7eLMslpdSF/nw+rMvx5Yvb4NzxtChPpaTKrR92zloAuExLrQmErO0Oh/eEsN1cPTzeI/DTs/Dqdqe1/vTTL/dd563LJWlju55yu7Sc1lZ2xmYSAW7M5QlZYEAWQ2itvnt8bOuCl/Obu3eWLNTorcFeddZoo53WCLC2qhGtcgwwdr4pdXt9ray/u3vY787jlb/aPv/hH/3bUvL1drPOadzundHeuf2z5z6E4yVxXoa+R0IBIFLMklK6vn1mnVuX9XB4uHt/X0q6vrkmpBynGBOS2va9xlZrK7n4buccKkUp2sZtt7uJ6/z4eBzHTdf1iHj/eNCX02FZs7FGEayZa4Obm9vz5fL4+KikVWalO4D0V/QJKqW7fkMgpFxhMdoua/bOGW0F8DwtnTebcVAKl2UO2k1aK2UaP8Em+PFHL66utlwTM19f3SJSy2ld12mac2neGWDfWAioMDYhIFFEKAiIU84vqO+Neiv8e3/2J3//N//WX//Gt9/cvf29P//Tru8rKo3kLVqtRaQ3OuZijKWar0O4vro9vPk8Xd/+T/7Wd85lff7ylbT83d/97eurq96YSitocFqT8P7FCxt6F+u8cqmNEJ01NTcW0VpprQAk+JCiv3n23BoK3XC6XOZpGvpBKVaIWjtAOE+Hhub2en88PHRdl0uNcdVaT/O51hqs8VY7Z2mel2VdWciFMcb01Oud88B1Wc/otk1kt98MfSfcnPWAEoIX4MOlLMULmR/87PXbuztF6uF0+dEXd0ts1hgQ7MIACMYYQqq1pJwPp6NSdLUfQjc4563z1nakrSA+f/EiBDv2vXWhghCBUqrCU04haCJEWEqeUu6M9prenR7/H7/9r//J7/z2D376424Yhi5468au22/3mtRTzL+1ttXoDe6G/tl2W2M8HB8+2NHXXl6lx7f/6B/+15fTQSuFIqSk65xS2jq32WxqrYrQ+t74ACClNFKaSAE+QatACq+ubz/48IN+3LGAs1Zp1/fdsizMVQSfApJbWXKaaq2n82MIzhoTUzTGLfOUa0KizTDovgs29MaGYbNd4yIA0zwZbazvdX/VjdcBo/ddSmutYgygVEKrjFviRRvPkgngsqy79eIwf/zsSimMKfd9V2u+TEfnutqkMzD0G6V0P4xxjfN8jHFRCgjau7uDcf766naOpcRLbaJINBFA0YSlSRPpnY05I8D783zdB0uktTpd5vOyfvWTT1qugJmU8m7o+w0jrjkZY2oukKp11vhu3O1jzA/n8z//F//8eL5MKZ1OD10IwQdQhK0Mty+1wtB50Ga+XLz3MTcWYX5iXUJl8c7/FfxenOuZawOaY/aGbm+unrZ0pfXlciJSQ+eUVjWnZV4EodZsjA2+W9bZOj8vCVCVUnS/2eWCtRVrXN8Pl/NZ2LNIP2zI9U4jiWqtTtPFGFfjpK0x3Ri6jnAZPWvVfeeXvnI4Htb5qLX54NmzUhIAGm1KnoGlcnM+1FqVMtvN1vn+/nDUWse0agIkHIZxs/Va2dDvBu+vtrv3n/2EU1RP9C/gIVYWQQQiuqz5ErPWCosyGlHh+/v7WsumG/oQrsYtAPR9v+Solek6U7ABixs2YdxhqCyyIl1yfnh8QOCgTQieCHtL2+evvPVGqZcff2kYNyDiXSXCp+yH1pqxVkDWuCI8hS4yADmrQaiVIlwaSy5lEzoA6ftAIGuMIvJ4PL764NW6RkVqu9vX1tZ12m6vl2XKOellrcxIhOuSujBezmcARqRlufTApELjuq4l5+JdiCypgcwXa2zwIa4pQ+u6TXK2oDfWcUtEEnwv0lg4hGFdkyK1lDX0XeddytlqhV5vN/sU47BBAeo6ezxenl1fiZRg8OHzn7bWLOEEIgJLLtyYkAgFEOZctCICaMJQ4Zyn4EzKiQByyUYprbj3IXETbRBAULGIIPrNOJ8v0KoiAOBlXbuhI27B0TzVm+cvNl2w3o/bvdam1WKMNsYKM7eWSw7BigChBkQhtU4PJGK7oXJDUvM8O+cv09lq3PZhjUVrVaogwPXVThG1BjFlpZf9bl9ymeep7zfT5ayny3mal2EcffDDuNvv98uy1lIOh0dntVJBa31/f6+VrbVY302pns/x+W6oIjFlENZ6LTX145a51gohOOvs5XJqlY31Kad1Pcc1hdCVRjGmPnQrYU4FEYjQ7FTMcV2mq6ubzTi8f/c2l+wR1BMC8BQ+/GQmANCKUm773jwCAogwkyJSSkCUwt5ZZawxZjPSeYkozDlp11lrfR/WJtqYw8Ph/PDQaWODeGd3V9uyrmCCM+pmN2pLhet0OT0FHgNgEWaRvh8RQKEwV+tcY0gxluWsu5FBcaspV2Pk9mpX4jqnfF4aQru9fVFK8c4QIZFBxFYLAPR9d//wcLpMwJVSTjHGOM+EqpQcuh6REWHohuAH5/yT6g1RCTcBJBO09aQtIDlnh2Gw1o7jxjs3jNsKyvvQWl6Xi1KmVZ7niVvt+6CVevPuLqWEdSEAUtQP/f5q3w99jHHcjK9evAjOHx8fUkxV2ChEAER40qIQIiACUW6tM4YUMggq0kRPXuJaalrnWiM3ETJ9PxBXMNb1vQm9iDhrnFFpXi7zdD4/Xk4HLEkpONy9FcEf/Id//fGHrz742jeFBUBqrcsyn8+naZ6V0l3wpDQpTYRxnblm9JukNwAKBO/v3pQ4p5yt60FZNIYIzpdzE1YaAYBQOWc2m+FJpeC86ftunud1XfT9+zdEljajNraU2nVd33W1IiIu67S/2sRYEAhAUJmU0jgO94fpt7/346+82t9suxhnxGCtW+aLc+56Ozjr5+VkrWchQNls9ooUAMzLrBURScurskPoAgLWWhhkHDd91xnrfvbZz5fLKaVsnbUaEaQ9RSADEqESIgAipY3qjXmS9xIii9RSgvfLuqZ13nYBSXErahgEOHSD0VhzhtZKyUPwJc2VwVtjFQIzMzDZx/enmxe3v/Z3/6eIpLVzTillREQpVWuelynnbK3tuo7Z11przZv9zZNOiUhPl6O2vooh3T3fX49j9M57a3OeC0NvjNa6tYagjVGIpLT++KMvvf78J1opVEpt99cAUguLwDBs7t6/E4EnV888T8s6W9s1blqhlhwckeTO6cZc0qKUPl8euy4Q0dB1TbjmjKhEmAiYW60ZAEspwcHYjyfCFlcEqLXO07LGOvSDMebtu7fzNHX9kErtjCYgTaQIC/7iVhIUJAKtCEV6Z5ZSFBEhtdZqq02YACQVpbRzJkcARYjkhgFQee+biDSDCLeb8bKuRinfdaSUG7ZsAlM53D2+/vEPus2otQYQrbTSJCJK+Sc+IOd8Pk9aGwDRWgfvkKCUHMad0lZblxkUUUpFker7LpeklMEKOedcigjkHEMI0zQJg9Z4dXVL/bB5/uoTF/bHh7fzdD6dDs4PMT1dp6SXZUkpd13nnM5xLjnGmEZv//o3vxKcvVzOzgeltNbKWqu109a0GnOOa1prySCAgACYczJaW6NSSfO8GG37biDElMtmfBIqzdP5sB17RKyNG3MVRESjtX6KsftFF0ARueS66RwSCQiAAIJRSrgJs/GOGESqC0Er6q22hC6EUqrUqjWFzXbYXFnnQnDXN/uWMmhdn+Tw4P70d/7D8XAUQGYhIiJUClOKKSUi1XWd9761erlclNKNa2sNAQlAaaVIOsNWcS65tFJbskYZ22vtmKv3vwgrjTFeLlNal3WZx82GlPbDsF3Xy+N5+sGPfvz567dKUzeMMa0gcH9/V3IzxtRatFLDsIkpLctEUL3vnHXaBBHo+54QnXfMteRobadIa6289wJcaxHBNS4AkGIcuw5QYooikBuGfgCi8+VonQ/D1llHhEuuIPyk1PsFGP3kqQfIAocleaW8MVqRUeS0tkY/qakRkUiQmbkBlxRX6zwJOO+cMcvhsS3HkmanzbYfxu0211JA51JLTqnW8/1diWciEpG8HEteAch7r415ultNKTUMw2azyTlfLpd1XZWiVmMtSSs0mrWmx8e7aV6GcUfApdRYkrXOGutdZ61flrnWkmquYEtO1PWbVpOhNkde16wVxZhevHi52YxIuK5rLklEHg5HADbWAkjJS5oeUpqte7rj03Zd71yw1s7TucSktOr7noiWZV6XKNxKKSULoXP26YoC99nnP2FpH7y83Y5dSlEpvdnucm3GeWdMqg0B9FOSB6EgMAsCEGJtLeZSWxuDM8oEZwihMntjldYipAm9cxoEAYwLxhilFLIwN2tUni+tNavVdhMQGI1PVXKppI0QrTFP56NCeTKJIKpSS2sNUQHQ67fvAZ582Nl7b5xf1hhz3u2uh3Gbc1GECKAUGW1qFUHDzKVURBRQgjLPyzyvANL1w5LW1io554y1RDh69dVPP9qM4+VyAYCr6+vz+cgszjmlTBc6Y7v3774oJRqN3bixzhnjELCWXGsNXSfCrVXfDzVOGmrf9db6cdxa43OFVOW4NFLGWkMICBTjer3fMfPh8aHvBgYAkHGz2/RdqrUAeI0GgZAI6Uk8LCDIkpkf1nXbOWOU04pBUCCVtKxzTAsCKxCrwJD2TjcBbY08+YpAuDRL1FrZ7cbaZI51mubL6QSApFUVevfZ58vlkZn77bU2mltdpjMAa6WMViJSShGR2rgP3dVuLyzzshAZFpinWbh99NFH1/utiDSGWpK0nHOZpsuyzIfDQ85pHDfX+91V71hAe++n0+H1j767xPLJV7/t/PU8T62VcbM5nY6K1G63974LPpTaWGrX+S70od/EWGOeCSpAJbLW2mW+pDVWG0K/my6PZGC73Rmjz2duMClioOrsWGu+TOebm2e73R6JjodHBQQACqCsS9/1V5vxZ+/uD0u+Ds5qKo3rX6XaNBIUEOaYGg688SYXRiR6ahGooTUENFaXVIGrsp11llmU1qUmaK0bfJZKrGzoP3t9v+QWS4nzMobu6vr65vnz8erWdhsRRMSc1lq45FWrK9OHG77KpeRc9FO6Wsu5pKHvUsr3j3cKkUUOh8OGxdjeGGo1kZLgvbUekI/HU61Za/JhOEw1sTIGNJE5Xy7L5Zxzffv6Zzo4rdT19ZXRWpFhBmb4yc8+m6dpM/bX1zvnfhE3LtyMViCsVQihA5D5chaByzT13dANV5fLAbh53yvtGE1el9P5FNdzStF733X9OI6Xy+l0fACoQ78TgePd/eV8dlo5rc9r2nrnjQEAoyXW1ppoeWJrkEFqY6PUuhb8BV0IiKisVUoTEjC0mj16rY0YL8ytIZSoAIxWu2GYl/T53aPvupTy28eHs3kcnH7+69/55q/8GrOUUkIXchVEDaTv7t4Nw6aU7JzXWo/jyAK1Zud8bdUYe7W7vUynuM7LMi/rvN3djputtNx1Qy7Vh+5yOa5LNNpt9vs5prVKq9UQ0nd/8uaQcP/qU6eUAnbOX1/feu9jjALoQ69Ix4o/+eLh9ftDbZxz1tq0kpblfF4KC7I0IojrkmtG7axSl/NjyiV04zSdL+dHRTz23lgzDv0yX8Zxu93uNpsNgNzfv+uHQTs3X45D728/+GBeE3A1mjJLYREQRHIKLQILO6Od1l4brbXVikWQAFGGLnS+241DMEaY12mqOcZlWpcLM5PRfju6flDWt1IAMAT3+t39JSbOJac0r3FNWSmcjw9P8UzeB4QnkaNS2l1Ox6dRwBhjjCYiRYhAtbCITPOZpWw2G20ss+Rca165FWe9It11Xmk6n8+Nq/N+u9ld77cf3Wyf7UaurP/sJ599erv94NVzTfjsw083+2dD5xDxeLh7agAA/Mmrm9v90HlrrWstOudTKt/78Z3z3bc+vnoaD+d5EkGlNKtitF7WSSkzbK9Ph/ci4rtx6HsAUKgVggu9MfrtF69zTFr7/dWzw93r4+F+3G5vr3fvWjOKMLcp5ave1SqCZJTygAoJCIEgaK1JxSaIeLXbv3rxqtN6XWbXD6R1PD+UmteUBMDNx7LOGmnY7XW3ocsRJRLhWvIcYyAqjRnE+1CFHt6/bi0jotaaGXIuINk6++LVB33ft8bH46P3gRsLCCDEnGuO58spBN91/X5/XUqrJY2bvbWOWTSR9+58Povwui5dFwgZEGqtOa85Jf2tT573hsi455/8krE6rUtw+vx4uLt72G+vmfPj4dD3Q+/N48PrzbjvN1ciclzK3Tn96vP9MPTWWm41paiVjTFqRQLAjK0la812//z+/ZtSa9+PzlnStMyTcl6ET8fHq6vbJiqu6+bqxdvXP9miHvoAwlaR1bTk2lsdrEklAyELAgIp0og757V1mhRUAWVSzMM2dNa11pbpMp2PJed1jWlNkf9ySYVzunr2cv/Bl2qpJKCVBsA1rp/NZwNArXHjWNrpNImA0vrJX9UY7x8OH314G0K3rmsIIcaVSD3tHyLgvZ9rGcfx6XZtRBqGHnGYl1kbO8/rZjuI8OPjYynFWfJOny7TOAy5FBDcX1/rj65HQAneO+cRWKSIlM8/+6xWUUbX0pQywzAiYNdvUlo63ojQzab/W7/8pcE/3WYmy7LUWo1xzKUhMYuzXmtTSjTab3a3d+9fc6u+G29ffJBTJFLzdFqX6ermlQJclqk2NjYcHu+H0CnjFEJvzJTrYUlXHjqrgVsPJjUAQq9N593pMhXgkrI1hhGXuDpS1nfz+f3xNLXWnjx+x8fjHPPQh+ehU9Y4q+pa8prWVGptUtLVZgTBy7qcLyc0Q4x5XVal0Bif4tJ3hhCXZRVp1nitSClqrSJCyRUJmMtms0Uka+2yXESYmWrjL754vdnuFenL5SIMtbbtZquQj+dz5x23uuascNA//+zHr159CroXqUjofXiKber6ARCQgARKSbnU2kSbDlE9PtwdH+621y/GfktEzLwsM4jKJTnvmQWprnEdBiUi83LiBsMwlrRO50fj+loUtLbM07jZi0hO+WnR6ser0G8ev/j5L+xhCNZohcgIvXeIQECtlCZirfEGTplSbqU2rq33VlrbDjsCfjie5hjtE/CEQMDaO2ft1fVN2F9x/rjevWPTZbiM47g146ubZzGm5YvPX7/9vN9+pQnEmBFhGMww+L4PRAqAiCCucZ5mbawxtpRijEYQ6vtay1O+eUq5lKyUevHixWefffb4cOeMnefLUzjMtCRtPPCSck25rOtyPIh+8fyDXNs6n/rrrTX0FP+QUur7jdEeRIzRKabH+3e903b/stRccvZWgyig/29Nb9IzW5Kf90X8Y44z5fgOd6i6VdXdJLtbzckQm5BlW7Bl7gwZhmFv/AW88BcjYMALLwxvvLABQ7Qos9kQSbG7a7r3HXM6Y8wRWmTpA+Qu80TGc57n98OMgXPO+0CpUErmgod5zCmUHwwJueTkffDBSSnNNEQf63bVDxeE0Wpz4507Hk+UUkY5Jaxu2v71YV4WDiSjDACcQMU5RlhytpJcEuQL5lLrqkKPT797PiGMnbXWBS25lhRFq5QiQrl51krnaFrAKOZKKoQxwlju7moq7TK+nl5KSQjzfl52N7d/8Ys/+O53f/fVT3+/W60IpYApQogSep3cBJ8IRYt1DiQXPKeSc8a4YMA2YE4KABmHaZ5NjElKEZy5u719fHz67e9+o6Wwzux2N8M0FUxfj5dxdl2jkZ+dDXS9vjleLoC85ERX1TgOx8NLyYkgVIAiVEoygGnyo00YyXUMbrteF7RFGAHOAGQaR7MYLlBV6xyKFDxGVDeNtRYjQBg45877lEq7un15/IYQEoOTQqUYKYVcoveorqsQrXNms3uDKUW4MAJQMIPrrB7hnGpBm92ebe7U6gZz9jD8X7i8AkABfOkvq+aeEyQpG3MumLfbe1QSJEC5rFlcbbYxhJIzpWzVNgbKPJxDSkmpcZp5pf/Vf/U//Zftf181XcmIUXoFfQEhKeUYnTGmbRogrNIMFTTPE8aYMW596We3awghdDFnAKIkSzGaGAHYbrcruaRovXMphbvbm5TSF5+/i7FM40UI1sRIFztVlWq0FFz1l/748okCXq22QOhVDyVFjTDd3X2wy3IdL8zLshirpNRaxhj7/uyDRwhbszAuMSDGaCk5Zpwx6zp16mdfpKZkMQOTVUoBMCqElegKypxzH0pKKUU02Wkah4JQyJlQyoDGlFHMmmNcsImwvv/yzU//qKpbTslv/vZXOf82p4wpizmU4F0MyaOEqRT1an9TouNKJTtWinw6XkqP1PY241hKRrj87Gc///Txm9PhgJg+HZ6p4F989RMf/GlYmrblgDFgIYS1dhxHKfjiQ0Fl0wgCuKqqZVkwBsHQtpWUFB/ctRFNGQk+Esrm2QrJN9v1+XSijJ/Ol7bb4hIlg8E5H9zNbgekp1ry7W6Xkj+eXo7H1/12k2PY1CsEFCEkhEIIzfMMwFVNGeNtW5WMc0F103DOnLMIIaUahAFjknOy1mhdY4RTQtbYWrWMUEowAZBCdN3WTRef4nmYL5MDDNvNzfF87vt+ngeEycvD987YiEABYYw5iFBKLhkBK5R2u33X1G3bMUaI4NdFJEJICfl86W1GHJObuqnqNWWK7W5MLBhzotPjr//+9z58AIIREJRxvdv/xc//6Pjp4+9+/dff/fY3RHIhJUKFMbbdrEtO4QcTPQHAq1Wbc3kd3X9cBqR5npumjjHFnCslESqHw2sIOQQLpDbWVxpiQdmlruZ1054v50qqw/GgBR3GKSQkGCmoxFKorlQpaR6Hh8eHm5vbql2fnz8hP1MuCpE/jHlzSQgpra5vFYDAarXijF6fP6iAlJUx86XvV12bCl6M4RSXDKjEZVlwKZWgIYSqqqVgw9kTXhEwhLKXl5dVqxfjD4fnx+9+22z3/ctLSinkEhFoQjiFK+IEIUjBU4wZpTG5kku32TBKi3cxJkD44Xj+dLq839/vb96jtktE5kRRCpHwl+NrwyllLKW0GFdp9fZHP2FK3e53/8mf/7Pnx2+meWnWuwKEc8EKitGH6FNO1z6uUqoUVIeSC8opPz89hBi7tk0pLMZWWnnnz+dz3bRCyFKQEEIIeasb7+M0j23dbjc3l8tlmscBEBMrIDg4a50TXFBK6evry8vTA+Nyt7s1ZqyahnMWrMnIRs+pqJq2yTkBAELIRxyjkxxXlc45T9PARbVedcP5mXHKhagKXC6nJToulOSMczWOx2ma1uuVkNLMl5iSW5YY891+RZD/+vuHu93ONs1JKhSzHXtGaC5hcN6nWDGqGaeEJYSs92YaCeGrriUUfvKzX/zf/+f/ATY47xe7MMpSDDrGGrhstiiluqq4oP3DxwVIo7harQsGxnndNfs3b401291uu9m9//KLZTZt10opCSE5F0oVxth7F4Kvqsq5kHOuBCqFxOjaVRe8//rr39RNu1rvKaGTH7XSnPJLf+66ddd1Of8AhY8hLYvpuu719YVQMi8jSYwz2rU7RlBBmc59fzy8Yky5kADIGRszCN2IVgczLeMFRY9zxJQjInBBpZR5XgimhMA8TynGphEhOIRyVTXzPHnv26YZe4tKHod+iWCsjckrpQgh1lmEYDx8Krxapr5tu/dvEc55v79nXLw8PcaSASMK4HNxsRCcFcepZECQClrmSTdKMGp86HY3dds9XyZUkPWeUUYzEsWDGVmOiBIumCCl5qgErO/eyGZVEITgqrZ79/7zy9BrJed5WpaZMT5NozGLUjKl9B8jACSEoJTl5AvGOaMYAxeiruthGNsO1XXNKOQcCSFK1cPQXy5nxigABoJjQJfLOcUgBQ4h3t7evxxe1uv9tDhGQTIyzcNmc0Ofnj4p3WVU9vublOLlco6ZOu/v7t7Ids9V7edLXHpEBVRdAaYlZbRDOcQUpnG4gtnn00moCgF5+PQJAP/oq58QKBhTQuhhWgCQ1lxK6b3JBZeCmarqpvvum98S2dZVfT4fjTNKVlJVqukOn74XlISQMcKpIJMyZQJhSlBx81hysTYKpe/fvPnw4fNPD48mlVxyxengw+js5fWT9YbqJq5ahh0uRRJE2n3EJAevBJVdczwfOePLYgCgqjsgQIACXGVi1FqLEMq5XIf/V8UmYwyjFEOgBDCG1Wp9jQCvywRjZ2Pmqmq0rsw8CilPZ3M5nxnFWmBEBAasFI8xVEqrSgMhx+PRhkITwimjuq66rjPL5GKs61pwbowtBVEq1Po2B2uny9K/UKaZbjjjpZAUkzULZfxy7k99//b2DoBijDEmxnrGNcGIso3UtqTMuNBaD+MRY0ZYWW/vdFV/BvD8fLTObjc7s3x8ef6IgSBMTYqloJISopBKSbnEUnDKnRSQy+vjQ7vqPuy2x9P5x7/401//za/AuZTSqtLOp18/vc7O3y8DAF63K67X+/2mpgkLeQU2Cs53d28oEd6HnIMQIucUghNCpoRSwgAYAKSU5eo3LMgH55yrqkoIZUYTY+RcBB+E5JTSZTHTNGGUt9u1kBVjDKGCMRESCsFEKsK1826Z591m/+233yBsU4la3a5X29ksNIQIEDlnGKNpGjjljLCExThZRskyjwC47VbVWrhltNPgzrOsV0K1KcUY07rZhFhy9kIIwfkXX3w5TXNMCOGYAX6YqTPWNK0PJvoYYqy0NsYuy0QIIwRLyZu6Xa3X3rt+GGy8kgERRsXFwDE13gPG67pJqVT721CKrprD8cC4/KM//2ff/N3f/v//5q98ysnOP/3szXkYn4OvUGx4JXd31eqGhEO1eWsxKSkAKu9/9KP17jZ6yxmxNthlqSqtpBBSppgLShjDVaSdYsjAKGd13XLmrDPzPHMuEIaUEqWMUnDOLcuSUlKqStHnHEJAKMdhOHebXVU3Kfics5IaMIsptt3qfD7kEE7HFykEoQAxZiHYlTpsl4kxWQAWeyVeFAA4fvs3j1//g/NFVOt6cyeUXIaTGQ9jfymoxFR8CJv1jgvx9PLona204gwI4HmxxvoQM8bAOJ3nqSBghFzVsYQQSslm3e73dwVDpSvKRAhxWSYTUyqZAEolh5RDSiGGnDxm5Muf/eLzDx90VSNEKGGMi3/53/0PP/39n64oJt52DP3hF+8+v7sjVWMBhuHy+O//reAoU56ip8ndffjs/ouvMMoEpaauuRAIKMo5LJOz7nh6zbmwa+JMSYphNs76bIwllNV1A0CnafLOlHIViNJxHK91lYJQRj+YcYfLiQACoIDwPA7BW84FIWRZpqauOVfO2++//5pQxriilVaUAufCWmONVXUFJbcaUir9OKyaOi+nwtU0LcMQmqatu3vKhuQXa3pGVC6FX+nSStrHYJihlABGGJeYSg5OcqKULiU5a4FQQBnlBIBRKaggIRQBGuMyDsdf/eqvhagWs7iYY0KcgADICOecUsLRW8k6Z6zSFUZQVxzhAhjdff7l//g//y9//1f/z9d/+zdh7n/+5duYyWU28zBUELfv97zdFlTaru1ub9///k8ZZ6UgzDhQJoSq6vry+F3MuWUaIUIpn6ZJa40QygWl6CSr+stJaS2VXnVrp/Try5NZzGq9opTEGK/TMClAKUUp8yGIak1wNvM5IK3qbp6GeTbeO2stY1zI+nz+XldtCOnSHykkN/QnQr54PZ9m62RVSsk5pe9fjoLipl7pt38iBLPG5uy9cwSAS75q18sy+eyLneqqxUAEZ2/u3gAlBKPz82+rdt81q3HqGQWt5bKM1991AepTSSkSQowxXEjAIDnxMeScKODL+VJKCQX5FOS1xVQQRqUmJBpzevlk3c8JBgBcVTVjXCnZ1HW72d99+DKFcPzm2+nwUCsm3t0KIYRUerNv7u7efPFVoVxpLaWajdGV5pxxznNOrtthAErpNE2MSwJwve5cmzwYXzUfJMYQc8BAbu/eLMscYzJmQQhbOzfNqpRi7SJ1+3oaaslQGGMIiHMp9OF1QQjlhJRqSil1JdfrtRDyfD7klGgukTGRUzTjKLgESgqGh4dHwfh+s7IhefVO0kWghDHNKaeUpFQhWOsClRohfJ3GhRAZF5QxVAoVdSzU20VwXlDOJc/zhDEQKAUL7xxCUEruVqucMhd8mhHC5P7+7eHhcegvKWcMEFO2JSBUas4EEBNTMT6lzJmgBBMCJWMpJcaIABjr1Obm3Yevyh/710/ffvrHv7f9Sei62d/cfPaF6tZt2wFhBSMzDc45Ibj3XghJCO3W2+PxFQOilC7LfHOzJ4Qsy3IFvXkf63bFKBunCTBCOFZV1bbdssyn05FShTEBQJTL6IwzMyfI+4AyANeM4XmaYkzeO0IIp8I64+yotS4lBx8WM1GgrORgnTXGMKarek0BBffdZtUwAOtmEDqkwiBrXVHCjTFVVQ/HT0JVTFQZIRfibGMlCBCgjFhrZLVdXDRmbCq5Wq1RQfO85IIJEGsnSoXgqqpVyTiEkHNknG/X22Ez/Nt//f/O1grKcsnX3CmkbGMEwHMqgtJ2e1PXlbfLMg1cNkCQGV3JuW6qzX5XVxUXYnt3J7rV+XRcdRvjFtW0VdVkRErJFAPCKAZPCS2ljONEAJWCYvAYl7qqgRDvnAHw3iGMMUKMkRDL8WQYQYJkVIq1JoYolZayNmYRQhpjEALOuPeBUTSNE06hXm8xwuf+3DRVyVgpfQVTUSoWM1PAxhrvDbSrTQrBmCWGSJlcpjGGsL/Zdd06I1JSqqkFQNb4eTbTNCGcMS7LPFZKV7UWnGAUMc4xhpxTSWkcBh8sI7nRspSilByn3hhLgAIBa20pkTGMC5qXKRdknfXeKlUzys7Hl5RSLgUjAABCiOCCUI4oiRhToKumxaj4ZSaYKCU/Pr/+9T988+++fYoxaqWU0s7OQ3/RVcuEzggr3SQMjOBSUn85jUMvVdc07TicC0LeO2P9YpZutem6DWVUCHG5DOfLAACA6dX7fBnGbx5fbEhCKlTi6fB06Yf+0rdtq5Qyy0IpizGaxcSQtK600jEEAEAoV0oBRhnjqwGyqqoUluCWeZ4JDk2zgt3uRkjWn0/WxRLDt//w168vD4yJeR6A4KZuaImKES54ydmYmVLinLM2LNPJjq84h6bWm7ZSSqX4A0BCSY1xTjkKIXNOx+OhoCKlAIC7u7umqWMq0zzGYHGJQnCM8cvzx6//8e8OLy8ESC6IAKWUc8a5EJWqBdOUVwUgpTjP5vjyfVXXGGPFSC35FfRf13WM4XQ6TdPY1HVd1z46BJgR4ZYJMPYhSt0cxkjVarO9SSFoJa/YMSFYKYkQwhmrtL6CwgAI4wKAUlT2DbXj5Xg6l5gFE1qrEJZx7BnlUkpnrTFmXqZ5ma2Li/Exp+BdSXGaF+NwiimFkAvq+yNnWHEa7aA5TTECELZq6vPpdRwugLyuayWk836eR8mhZB9TopQJLpx3fX/CGHu7CKGqZhu9nftXN58RzoRRIQTCSEpWUOFcK1k3dUOAoFKUrIDAtel+bYcRIhBCdu59QM/Hcej7j99+PS/mmjhdv/4AhDLGpaRcCikTZSEYxqlPyDrrxuOmUV/e1P/0p1++ffOWEHh6+phicc4Lpba7vZRSEhyCCZhQxgkQjOlg8mhL23WMc0xAcK61Tim/vh5KRsfjKxf0zf2tcz4ERxnjXFJG39/tCOToDTBadWulFAEIIWJA683GeZ9zrHRr7bzME2G82+xTjCEEKTjgohWTSjo3Y4wYk3Y+cYjterupOeSC627rfVC6prJZ3XyOgBBChZAYQy5lXqZpnnMuSsm6bqSU4zhN1oXMZHvTbu6t9Y8P3w2HB0CRMZpLHoaLs3Ycp5hSyinGyLm01qSUrpXKGJO18zLPIWVKYb3qhKw/PX66to4xYEopZ5wyCphgoIxxinAOeelHgmF79x5jmPtXMw/GBq0456zvz1fLhFKKUlrX9aprlJJKSi70NdQMwVrrpmU5nc455VKgnxatq5KLVjUhoFQFV9S6986ZFLO1pu9P1rm26apKmXmw1gohMCEhhNPpWFDebrcANKZ07aw0TSNknTIKIVV1pTWv65WzS84BMDu8vqaMIpaYybapoRSEqKCUbXa3XNUFk3mazw+/YZRZE1KKpeR5XlIK3vvVekOAWGeneXl8/jhOC1C12b+r69Y6cz48mHny3lHCU44EUIp+nqZ+6H1wizHn84Fz4X1AKOecKePARKXFbt1SRpy1uCAMGAihhGqlCaEFIQDIpSzBex8uh4NzXlcdYdIlMi1O1TWh1Hs3TQtGRHKxWW/NFQgKFBMWc8o5/rA8ZGyl8UpCjnGexuB9yihnNI0nShAAwhhRwlApJSMfQgzx8PxRSwmEUMY516fB9EMfUywFnc/HmLKzRinVdV0IXgh9Oh2jd4sxz6+vIcYYA2Pycjl7NzNKf/f1byczF9YAbxCiiSqKMTLL5L3jgpeCKODZ98PTb3Z3X1jsCEUYY0IoQiXFTAjxwaUYlGooY6WU8+XAmZCyVbp1djqfz4tZ9pt93dTGOGcmG6K1PgW7Xe/mZXHWSSk363VBueRCCMeYGnPq+6NdDAZMCb323QmGhBAAJoCcddM8JUqiM1c0IiG03dxzQe/fvAXAT0+PzkUUffHHUslQpOBE122KPqZICHgfcglmmWpKFArW4IQRYXyzalPOGLNccn++1M2aC34lbwJAjCEkvGs6hMH5zBjdbHbTPJ5OJ4QKBiqkHPoBIdBazzOzdgFCnTcYUSkl4zxGbMxlnvvtZhWsV1JQrgmhJeV+GLbbDWCMnFkoYZTSp+eX59cnVW/e/fSfIyBScq10ioEzQqlgjAshnZ1jiFxUjPEYrWCsJDeNR++crlb3959td7fz3J8Pj8EbwniMcbe58c7m5KWQbde27fZ6wRqHo/NuGC4AWKsqxpBzKTkTAMZYTNGnWFIK3vXj4IKXlEAKpATJSAqm0rxpmuveYZ6XnFJeXlBc5mmgpFi7CKEIZVVVYwyUMh8iJiQEa42RUlW6vfLbQ8gZ4ZSRlJJReu0/pxIRygClaqqcszX2H7976RdXVTVGsMxLQbBab5uqmRw8vpxiTG3bARCt6mWxMbq60oypZZkQCpyyy+F4GUelV5ypSgp//h75yVkDAHhZFsY4JVQKpqXKiCaimBCllH44z/OYkovRp5Q4Y8YsCF0hnYwS4dystGaU+eAPx9d+uChZb/fvqm4XcrLzkL2nnBMuuJDXg7WuNaUQ+0/ITEAAqCAgldJC8hhTTAljHHzw3paYrPPTPBvvtOCbtis4j8MJADOhgjVCcGvNMA4EGEFFV211+5OCubPLMJxjDNd44IpfZlxSRoEAFgrjRADmeXLOpBgoZUrpuumcdyXnEDzGQCknQLWUCGHOeMHF+Xi+9JSxpm4rVdW68iFxzgDnh4/fhODbdh1TmKbeWqN1E2JgDAEGM4/DPAFhJsTR2phTRGDM5KwFlJNzPuWCEMEIO2dOw/jtp8drr2RZPOMKIUQIvca287TElFO0dukpBSH1MMfJM+9LSWkYhsvl7L0ByjfbG0ZwjIkAbdstRsQt4zSNpZQroLXa3AshGdepFKF0111JVJkSErx1wbsQ+nnspxGVLChzwa3v32z298N4JJSA0BjI+Xx21qUU6raT3RtEa1XXlAkASDFgDJwLAFRKKiWlmKTWIabzZbmeVRgRZ00uKIR4zTiv1DVKGaXCh1gQEAI5h7cbRXIwy0wpIVykUgjgnMK60WnpT8+f5nmSQlLKcy5Ka4QQg6gE//jx+8ksmHAlNWfiirsyiPFqlTMCa2bjFsBlmKbJ5Uo3OSaKkXM2RM9Edeln70MuqFu1zttxHEKMnPP+cuyH8zwvr6dlmNJVvapUgzCd5/nw/EQBcsEFoarWCBVrJmfGZZmPlyHEhKs9FSp6X1KQArquub3/jHGqOJeM1lIKSifvYoickk4JVHLGpdvetKudVNX58CSlyrm8PD+nlKWQMUXrnLUzY1dNAKVMpJRcCJxzSpm19nK5YIQLgpCxc8F5W1BCCOeCvA8xZGsXIFByjiECkJxzCMlaG2Nqa4lxYYxLLvpp7hfvfUAYWzMvLtbdNsby/PKgtbp/85Yy4UxPULKL4ZxL1abgc7K1opphVMrpeIwxUYLBGmONu7Z8R+NDQptVu9+2UirOxHmYXw5n51wMnhCac8YFUUoJoXWzbpv1vMwUO8l8yT7nzAVlpBhjKaWM0WmegVJKwdjZ+iiqrmna7bpFGDX1Kud8OL4Gb7wLNzf3f/Rnf/6Hf/hPmkpN09xWNac0hrCpqy9v91/e3FSC6a5r9/fTPK3X+7peXc4vp+MhpVxyRkDP41w1K4yRc4ZgxCi31s3zIqXGQI1Zcs7GmHlaGNep4Ien5+AdKklppTgjJCOEnLMxphhTSunKrM4pn879ZVw4F0IIICQjklNO0Y9mMWaZl5nrje72GGdjppxDU1cERUBpmk3IIGRDmWo3d4TKGAylJMSodaNklUuhPvqUSsaFEti1KqcgmBBCOe+MmVDJXVMzxjHBAOBsQBgJUeeMuOBN0zJOg3OUyX5YQqRAfEx2s91opXJKzvu67QDI6Xicp+lHP/qxriqESkplcfOlPxtjuraklG5vb37+85+R7P+3v/zL19dv57pmjHFKVk1LqaCEvH3/7v4nv19vbpfFAMbrzfZyysfTMWccg//06Wm97qSUxpoYYkkREDqfe+fMZ599hTEBIJSSqqpDjOAXglKMHgPmnFd1VVCRiedcnJ9TSs46QmkIDgOZxv7p6UlVXVzOsqoL5qUUKIEgOo0L4OJdrGvNGANMOd9TSpd5zDGcLwPChPAa4zQsS57D240mhGMgDpcf/fgnKeFhHqHkQii5v3tT181+s6oriXEJhZzP58fvv73ddu/evkEFJWc459bOCGHOOQaQUpeCL+fh0vdc8LquKi04Y1LoFGJV1zklSlhO6HK5eGe8mRCCZVlCCBjjmHxKnnMRU+i6BmPEGP3w4z948+4+lzRbkzEGIKmU5/N5nPs/+xf/4pf/xX89DMNss3NRSE4pKaUgKLqqnTOcU+fT83E6nQ/jcAaUu7omQOZpAoD1eq2UEkJgDMFbRvGbu9ubm33dtDlHZ00Ifp4GjLAxC8KFMSakVEpTzqUUGOfx+BDnY9e14zS9vDxfzie3TLXWum4RKpQULsSValxSOF8uLmagKnhHaaklrZUoWNqAMdDtZu0CfjzPSjA6DT2jTGkpJB8vZ0AFAaYYz8Mw9peSQ1yOlEq+3mCMnTEIY0IxYXA+X3JO333/reTszdvPcg6CU8aZcy4XRCkf7BFhWlCJIbx5+9nQXFKOHGjOJSWrVFVVzllXSlJaxRgIgfV69wc//8W/+dd/hRGmGGNchmUWALc/vv8nf/rLqtu+PD9IBk1bI1ROp2PKSEpSV/rN3UZr0c/GBnBz39y/oVJjDPv9rXe+pCiEcE5M05RzzjGsVu1me5NSGMeLMSbGjDGmlOWCnHNa11c5Nca4qir29q1zwbrH5GdrPcG4bdcfT6attVQak1QKUEqdtX1/LNdJqI9Nu0qpAABjUDedNe7vfvNbqdSbjZLi3cfDGAn/qu1gXhYASglhBHwIfT9QKhWDaRmMmVMIcT6E6YkyklIIwaWMMEIFkZTSNPV11Wy2e+ejcXExdhhGjAol1DtvzXy9x1LCKGX73S0hGAAOr8+nwzGGCJgQAldaszEGlQKAf/x7f/Du/VuK4o8+vP3s/fvZLBL8n/7Jn9RNk4IDjCFbychwufiQuJRNs5rmUSvOKJScECpdt2vaNSEipkgZCyHOy3Jl2itVMUalVrqq+v58OBysdZzL1WotpVqtt5vNtmmarlsJIeZ5nucBY0Qpzyk223u1fns4DT6EEKIv1GTez04Kwhmf58W5ed02BEg/DJzz/XaHSuCcS9k8Pnx8evjGRESFdm5+ePjWR6Q4S9HTxdhudWONVboCQpd5qJtWCvXu/Rfx9k5UmuDPnQ8lpxhDjAkw8S4wTKUQdbsVaibYj/2JcXk4Xihgsmql5NZO0zgSVs+LqXRVckkoIIRyRtZ6AjCOp5IQKohzwTg7vj7bZYnZvHn/4S/+m//2N7/6//75f/4vP7yc/vf/9S//5I9/+uHLr/rDE6+7jBCjrCDc9+cQYt3Iplmfjk/LYtbbe0zSprVtVYfgC0LWeA9pcWZ2vmnb6ynKOWeMWmvn2TRNV0r03i/LUgoOIYbgtVJcyLZdl4KNWQ6HQ9O0UsmSOYAWbOKcIBQ3TaO1qhQtGRljrV0YzrOJESFddzF6yjBj1DkTffz23/87Khhp3tqY32zuT4fvFJMSO2MijSkTygrC1/Xl7f1bpRTGcHf3LiVLCU2iZjQxynNOy2KreoOptNZP0yTbe2BIAjtdDjkDYzRl+Prj04+/fN9pXUoZxrOuOiE4huy9994xqjfbmxhMKQgY5IykEozy6fRMCfEpYUL+6S//07u7u9u3HzafZYzKn/3yl1IIzAXnYtWxqm1zTv3lEmMEQN6OjDHr5nlZGGP7dTOOw5Jc265TjC7OdVVTKr33nFNCiLXWmCWlrGS1LHMpUUqRc0kpXs4XITglJIQ0jVOITumKcz5NY103znnvJimAEqakqimrKlwpdTicQpijW6ZlDhnX3Y33S1U1GAGlNIRsrWk2Oyk4bda+gI854+q206UEDEBDjBihbrVCJQOwkEJL4fn5UTX7ut1Gdy4ZJAcu2DT2ISapK8ZKCEEpWXIUDFDOlNKEaNOpGOI89wA4pYAwolQAQdN04ozruoshz/Nk7PLdd19/9v6Ltl2N41DVuuScgAHF21ozikE029v7brXp2tVq1Va6ShkRwLkUQighEIL3MRJCMcJjf8wIKKvHcbq9u00x5oKFqFHBhBJAjGBy/UhV6RCi9y5450M8HJ5ubt8hLEoG79NVY0QIi9EzVpWSTseejcuXX35ZUnp9+Pb+/Zcx5uN5mN24X7dQEqVinMz5fALsY/A2RKB6Gq+o3I0PUesqp9L3588+/J5SXAiVShmGMRbQSlGicinUWX86nykTuqrOi4OC4sPHkBKa55gJh4xydiVhXKy1GEjKmBTMKFl1jfX+Mgy7TbNe7RazJO+tmW73W6V0CJ4T0LpZrdbGWe8tpaSqKmOWpm6229vHp0+EEmOcc24xS0YECLRtg3IsCG23e8Gw1orSu2mal/myqhVmKniXczZ2KQWUVlzWx3nRWnBWffr0iQAAr3rjOowQwkLwhPE0L9SH27tt8GGa5qbW/dmk9EPutNh0drOgVEq5Wq3m2ZTk+8EihDiXzrllMU3bYZRCdN5n66yQFS4JAVibnp+fx+GgpOCcbW/fm9k5M6acvVtKiVVVAYEQwvF83uJutdrGaKdS1t3KByuFds7TnDMhPMSUol/XAuXy/beX7e4+BVuSdwgAoOt0Kcg7U3IJIZaSOEVCqGG+DPPUNSols8wXQitj3c3NTV03w/nFW7sYk9xF6Y4DRskrJbSWBeWmaV4PryE4gIxRfn1+Cs5RoQhhgwk5OsaYnXqrW0woIbTmJPRHunnHCEghDocXSllVV5QRLiolpZRSa365HHe3utUVpyjlACBma0J0N7f74MPpdNS6woDrqsqFBJ/6vo+FA6FVVc0mUIYoJ7jIw9NDwezHX32GUQ7BOTtX9WpZxpfXp5KR5IjSlY/o5enTOJw5A+991W4YU6xhUfP+cj6fj1XdEEJyTozScZ5zaWfncf6hZ3GlD+cUaVXVm22zXu/maeIEgyBVs4rR22XgTAiIWG+EkKUU7xwA9m7MrmScOZMMoV3FAAgqSMhKa73b7VLy3ltrLZOVILSgvHjvl2m4HJXSTDVKNwWhu7ubFMvzy3PwPng/TtOay1KyWWaMf4Cdj+NQd2sueTTgEsohacUxgZxyiIEQ+sNLV8JSire39/MyVUoQUIRSY6YYI6V837Z1Xf3ud791zmMMbdtiQrQWizExeqkFweBDQAgNYy+4EEIpXfuMCAGM0Pk8YADCuBRKKeVspFTEiM6Xy8Pzs+b40+vh88+/KgWit8u83N7eKKku/cXYgPCMSiSACCCtKhto8oFQ4ACLy4/HnmRLpdSAxTBc5tl0bSOpxKiM45mgvKQUyryqdzkV57x3PuXsrCVCZiAfH58JYIWt5dXL6+Pd3VvOREk+xXw5n4b+culHXTWr9bpuupyTNfM8HPvpCeBYUNmsV1worSTCBQPmUoeQ5uG4zGdEVNdom4TPsCzGu4Vgord3wBRlaFkW7wNnHDC1i4kxheAoo7qqZSkYIJdUwhUqUZgQ6/XGe+v9D38CAANhPMQAhBAEOEcuKwzAOby8Xijh3vu64s758+lQ6SqVQgmJCWGUNpvdp0+PhIpxvHx6fHQ+SYp3+zspK04Rwtg5O4yTt3P2k+CVc36320zT4J0dh1O9aSIikjBGYBmm3sS1wIARXpbx8PI9ISjEfD5fqrqe5wlhfHv3rtl/yCghVFApMXgAMgzLbExA7Ne/ewqINft307J8/fV3PsTTZbj0P4ynxuEUU6KMvbwcnh4fnHWUyWb1drt/t97tdd0Y6x4fPh1ePsZlMPPIKGvb1ifAmHBGgIqq2SCg59Px5fnJ+SCULskDzs4Y5zwBHFOeFzcMF2tnIYSZ55RyzsUa8/D0NC8uZooxjzEcjy+Msqv5czELYxxjLDh3zk7TAIClVAghzhXjEggoIVftinM5jhcKBYCcxvl0OjprtKzG4ZJzaJVkOFLK727ftl1d11VKqWkq4/3kYR5OuSClNWeMMQGAYwi0eJQsQuXcj3aZS0m5FLqYmXOkpcKYjOOIcdRQ7/d3/eU1Jk+JwDgLKTCgmJNSK4RCfz5ONiWEdquGUdLU1VdffSUYtSFbM1e1ruoaE66YqOr2m+8fG2swghBDpXXKGWHQuhZCNqs8jX1JNnmTM0IIrHVKVUrX1lnAONo4zIvgAoAFH1KyUu0v51OMgTGSicxcsoJSjqfX5/H8ur55p6UqBSldTza3NLOKjMN4OvUYYcE4BjDGVlUtpQ4utG3jQ1iWhTJ56c+cS4ILF/J0PhMAJWXmMiQsALngYkmohGnqn14edpttPxz3+11TrbmQQtUxGoILFZVfTFXXqvvxdw8vSqa2rrSuMYbzZcjoWXBIqpoWSyheM5bcCG234kxSyghBQBDJ8fT6oLR++/4rhIAxyrnmXOaUc0EhharimEsp6H/2x7+nJX19fZ3Gy3bd9qfncXaoYKV0SjEhJKTKOW62WynrmAvCpB8uMcbL6fXl4fvD88PlfAret5u9kAqhgnCOKVnrKOWAseA0ljxa39St0srYBQBzJsZpKgWtt3vKVUSkAGnrehku58Mj59Q6C0A45zF4gmMleYiBUCaEbFad0rUQ4to8klLsdtuu66y1xsycciV5CG6ax3GxOaN+NAg4pkwKRnBBBRtjD8cXweTlcmGMK9VYO2GUg5tjSCE6523ws2SZC02BPB9Oxpiqqhjnl8vFOTsMlxgiRqUSBIVZKfkfAJJeR+/ocu/AAAAAAElFTkSuQmCC\n", "text/plain": [ - "" + "" ] }, "execution_count": null, @@ -561,7 +560,7 @@ "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAAAAADmVT4XAAAK3ElEQVR4nO2a2XIjubGGM7HUXlxKXKTunvEaYb//uzjixJnweNyj1kJRXGqvwpK+ILV1j20UWzo3R/8VCyQLH4BEApkAwLve9a53vetd7/r/LnT+ISJDxhgyxMOfjDGGAJDIEp0MINx/KT0vCALP8yRHADB1XlQaEK3qu5PrdwdgXhin4/E4SZLAZwCg1l+u1i0xZupSmbcHkOF4cjafz7LpJIkYALSX/+vLkgRXOXVvD+Al02y2OD9fzs6yUcwAoE2tooIE731Sp3eBCwDjXMTZcrFYLBbzbDoNAAAgWmxLqEiwLhEErdLmJEt0AEDhB+Hk4tPH5Ww6GaVpcCyP5l3UAmPdbSjEvqybU+p3ApDhaDL/3R9+vJiEoR/Ix79OWaaBYTMNuFitqddvBhBPFh9+/OPvz1POgXN79AMsDWaECG3MCAVasz9lDBwAuJ9MF+cXF+cL71BgDxVxcRiMlLpWMy6CsO6NtQSDOP47AJPRZPnx43KaHOsHJAB8cqIsyVoIR/PtZr8vqk4bO4TAAcCLs4tPnxYj77EIX7pwObb+9GNRblZXV6u8ARgyJ516YDw7X0xCdnj+tnk84vFC9f3my09cG0t2QP0OAMhlGCdxIMAygG9eTgAYBikhmF1qyn2rzaDZ4GCE1hpjDBEyAGAEQPDc0BA4AAcAOa9nZ9Oq7/vXBbC6LffbyPfDY32HUaCHhyfxMJ1kVdsM8kgOAF25XaVChsljERHRcSa8sEcWjLKyKvJXBujr7W3Ag2QUPpQYq60hYMgYZ+wRwIKXTst8477HcAIg3Rb3gRePUs8/NN4Y1Xa9QS6E9D2PAQEhAhAPknGehpINmAcOtNSXW8GD0DM+AiDnYOrtJlfohVE0GvHHVyD3ozSNQ19odwIXAFUxrbnAOmAIIgiE3v36+brFeDTNlhgGj7/kMkqSJPS9AZ7IZbx0Y5qeMSpCDsxLRoFZ/fK3nyuczJYXEE8B8DgrmQziJI4Cr7OEjv7YBcDYvlJcQBlyZMGkic3t55/+p+DZRQvjc/NsFhwAfI+7d4ELABFAEYSiDTiyoKgSe3VzsyKgIC0b9Wy4uQySNIkCj6Pzkug6ZXS1wcpjiP5ok9DqZk8A1pinlY8AgHlAzSSNfKm1K4HznO1zk3MGKJM0wv1dDQC+5PzBCxACAAfG+3ESeV5vXQfBGUBVHUcEEEEUsnbXA4ShL4V4JAAAFFymceRLwY3rRHQGMMcmoRf4XDcKuJSCM3y+HCAgCMHZy9JXAngQdbrjVpnDOvDVd2Rt1yttrPumaDAAgCEGBg7tfRICkNW62xdV2yv3bdkJAED2WxNHACCjmnKzLapOufviUwB+c4YhAJi+3m/2ed0p9yjppB54+vSyHtu3ZVHUbW+sc+KBnQDwULs1xr5oqunrMi/rrh+wGH0nwEt7N31TlmXdKvvbw/TKAN/Kqqaq6qZXAyKT7wBgQnpS8GeDbXXftm2v9P8JgEEvGY9j7+kNiAdfNCg0+x4bCKeL+TTiT+/iQkjBmbsf/j4Ans4/XGQRf4oQuBDiaXl6c4B4ujhfPPQAAQBYa4noNxaJ/6ATHBEiImcsujhfzLPUPw4BEeimKouyVa8cnH5TPeNemKRx+sOfP81GoWQAAEjWqHazurq8vq/U2wIw4cfZxYfldHZxfpYEAgkBwOquzVe//vzz7S7vXzdB8TUA98LxxZ//8odFFMeBlHjoAaua/P7m889/X3ftK0fHXwOIIM2Wv//LXxeCIT3am+2r/d3t9Zcv24GJ6+EAfpotzs+Xi+jwfKzPtvndzc1qkw9NmQ6ehjzMPvzuh8X4MSA75u6ru19/+bzKB3X/SQAsmn388UMWPHb04YMu15f//HI/PFs6HCAYzRazVGiAQ7B+KLZtfr/e1sNz1sM9IROeJ/kh9qRnFkcAyKUc5AbhlG25aop8NCIOcEhZHsTD6XnLBKPWDDvAGQxgm81NIOLsAHDIVyGQmHzCZBJzC+2gPOUJAPXaQz5ePmslAYCcisl86ulW255eN1P6NUCz4xjMP9YpPNsVk0jjs4nX7XetJufA9DQA1eT+/epmrIU4+mEAAIZcYjk7yypt9JDVaLgR2r4t7r/EdpOOx9GDyRMCQBCPslndd6+cqPxa1vT15ldbnC8/ifCxlBAAZJzN67Z65UTlVyKyut3banOXszhJnn9lWTRZNPV++6YAQFa31JV5Bck4ohDpyRBENKnaarutevdTk+E9YIxhPZOVklFgz2KPe76QB6/gRVNtmqJSVa9cc/anBKfGKIBKITP72TiJkzThh42hiCxD1XS0zcu3BAAAALVBatazsyybzsE7nOWJkAfSdAp8VK5T4WQAsAV2u2w+X9aWB+EhLvC9SKi2t0w1tePOwB0AERnnHI87ACKm96ppmq6zXPCQcQbIAdKsarVtm7Z0i9FcARAZl2GSRBLJEiDavqmbsmvqcl91XTeKwkAAAHjxmQJmDW3bziVMdgZg3Iumy+VZxK22jDNTra+vd01Z7tb7qqnnWcYPhuCP0ZNEGOxzcvDJ7hcYuBef/fCnTyNplOGS6/1n2dZN1+SbWlndWS8GAACSqR9woxSibh1S5u5DwGU4Wv74p0ya3nBP6K1stkXXK9WLeJREo8MlBiLBA6mKzaYsPf7f3joAABCZCNJsfnboAaG9ervrZdBqlAysedoGIfj+y8TFqwAAMi79KE2E1YZJZu38hz6Y3RW1TbNRFMhnNVqj+67tnKLUAQCAXEjPY5ZZxpHCTMlsfbvZq3A6n6ah9wTQtVVZlnXvki0c4IgQkAnBgTOLDNAbYTzbzO+2rZdk03F0HHEi0m1dFXlRO11tGeIJkXEuGRwOasFDESZJEKUtD0ej5BinAxAZ3TdVWbklCoZ4QoZcPMsISSYERxl3zIui6DFbRdaorinzsnNKGLvPAkBgiM8CGeQQIHojhcLz/MDnBAhIYI3u6mJftuo1XTEAPEVih/cSMO6zQFlEzrl46ByyRvdNmZdu93qcAYgs2eOG9yke4vJwVvPs9ICs0apr6sZtV+QOYI1WfdcpzoAdIV4GlofeQUREssY1XesanJI1qmubuuqOloX422EociE96eoHB0THVquuKYu8/ncr3NEXI+NSuHti5yEg0n1XF9tNiMx/LDxUSc8ejsYC6NqyATag+3q3vuGqi0OOCPRgdi8Gm7Tpq7xsnKMz92lIti/Xl1hm4zT2fd8XQiCDx/ssDxah6v3m5vKueu1dMQHYvriF/HqSpqN0PBnHIePwOAAP9VO/vbq8/Hy5e4NtOanS1ndJEqeTs8X5ckqMPyM4StXrf/70y9XtmwDous2lH0bJZFH0VluAiD26RQQAoL7c3X7++z9Wef0WgQkZoxr0g6jqLZim7XsdiOMtGjLWKKXb/Ob6+vp22zif2gyPDVttLFlV3G3y5WwcHxOmSHVZ1W21v/3Hl3VeK3ptP/BMpgNS1Wq03lddTw87T52v7/dFvl1fXe8b436p8KTgtNPNTob7RlvDg6NbqjZXN+vtbrvd3BdD8vUnAJAx0ABIwyUnzmEMAFCsri+v7jbbXV6WzgZ4IsBRancXsq7NVyMAgPL2y+XqfpcXVdO97YnJk9qd6Her6SgCAGh29+tdUdVNq4acmw64Xf+teBRHYRT6EgBAN3Xddr1SA6+Ufg8AcsYZe/AD1hpLZIkG3qp917ve9a53vetd7/oXsYyZrAWZvNgAAAAASUVORK5CYII=\n", "text/plain": [ - "" + "" ] }, "execution_count": null, @@ -626,7 +625,7 @@ "#export\n", "class ExceptionExpected:\n", " \"Context manager that tests if an exception is raised\"\n", - " def __init__(self, ex=Exception, regex=''): store_attr()\n", + " def __init__(self, ex=Exception, regex=''): self.ex,self.regex = ex,regex\n", " def __enter__(self): pass\n", " def __exit__(self, type, value, traceback): \n", " if not isinstance(value, self.ex) or (self.regex and not re.search(self.regex, f'{value.args}')):\n", @@ -711,7 +710,9 @@ "output_type": "stream", "text": [ "Converted 00_test.ipynb.\n", + "Converted 01_basics.ipynb.\n", "Converted 01_foundation.ipynb.\n", + "Converted 02_utils-Copy1.ipynb.\n", "Converted 02_utils.ipynb.\n", "Converted 03_dispatch.ipynb.\n", "Converted 04_transform.ipynb.\n", diff --git a/nbs/02_utils.ipynb b/nbs/01_basics.ipynb similarity index 61% rename from nbs/02_utils.ipynb rename to nbs/01_basics.ipynb index bfdb4986..058447fb 100644 --- a/nbs/02_utils.ipynb +++ b/nbs/01_basics.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "#default_exp utils" + "#default_exp basics" ] }, { @@ -16,17 +16,7 @@ "outputs": [], "source": [ "#export\n", - "from fastcore.imports import *\n", - "from fastcore.foundation import *\n", - "from functools import wraps\n", - "\n", - "import mimetypes,bz2,pickle,random,json,urllib,subprocess,shlex,bz2,gzip,distutils.util\n", - "from contextlib import contextmanager\n", - "from pdb import set_trace\n", - "from urllib.request import Request,urlopen\n", - "from urllib.error import HTTPError\n", - "from urllib.parse import urlencode\n", - "from threading import Thread" + "from fastcore.imports import *" ] }, { @@ -44,9 +34,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Utility functions\n", + "# Basic functionality\n", "\n", - "> Utility functions used in the fastai library" + "> Basic functionality used in the fastai library" ] }, { @@ -56,6 +46,16 @@ "## Basics" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "defaults = SimpleNamespace()" + ] + }, { "cell_type": "code", "execution_count": null, @@ -125,7 +125,7 @@ "#export\n", "def basic_repr(flds=None):\n", " if isinstance(flds, str): flds = re.split(', *', flds)\n", - " flds = L(flds)\n", + " flds = list(flds or [])\n", " def _f(self):\n", " sig = ', '.join(f'{o}={maybe_attr(getattr(self,o), \"__name__\")}' for o in flds)\n", " return f'{self.__class__.__name__}({sig})'\n", @@ -163,6 +163,23 @@ "test_eq(repr(ac), 'AnotherClass(c=some-class, d=bar)')" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def is_array(x): return hasattr(x,'__array__') or hasattr(x,'iloc')\n", + "\n", + "def listify(o):\n", + " if o is None: return []\n", + " if isinstance(o, list): return o\n", + " if isinstance(o, str) or is_array(o): return [o]\n", + " if is_iter(o): return list(o)\n", + " return [o]" + ] + }, { "cell_type": "code", "execution_count": null, @@ -174,7 +191,7 @@ " \"Dynamically create a class, optionally inheriting from `sup`, containing `fld_names`\"\n", " attrs = {}\n", " for f in fld_names: attrs[f] = None\n", - " for f in L(funcs): attrs[f.__name__] = f\n", + " for f in listify(funcs): attrs[f.__name__] = f\n", " for k,v in flds.items(): attrs[k] = v\n", " sup = ifnone(sup, ())\n", " if not isinstance(sup, tuple): sup=(sup,)\n", @@ -275,10 +292,10 @@ "metadata": {}, "outputs": [], "source": [ - "mk_class('_t', a=1, sup=GetAttr)\n", + "mk_class('_t', a=1, sup=dict)\n", "t = _t()\n", "test_eq(t.a, 1)\n", - "assert(isinstance(t,GetAttr))" + "assert(isinstance(t,dict))" ] }, { @@ -296,7 +313,7 @@ { "data": { "text/plain": [ - "<__main__._t at 0x7f2feeb9ec10>" + "{}" ] }, "execution_count": null, @@ -306,7 +323,7 @@ ], "source": [ "def foo(self): return 1\n", - "mk_class('_t', 'a', sup=GetAttr, doc='test doc', funcs=foo)\n", + "mk_class('_t', 'a', sup=dict, doc='test doc', funcs=foo)\n", "\n", "t = _t(3, b=2)\n", "test_eq(t.a, 3)\n", @@ -326,7 +343,7 @@ "def wrap_class(nm, *fld_names, sup=None, doc=None, funcs=None, **flds):\n", " \"Decorator: makes function a method of a new class `nm` passing parameters to `mk_class`\"\n", " def _inner(f):\n", - " mk_class(nm, *fld_names, sup=sup, doc=doc, funcs=L(funcs)+f, mod=f.__globals__, **flds)\n", + " mk_class(nm, *fld_names, sup=sup, doc=doc, funcs=listify(funcs)+[f], mod=f.__globals__, **flds)\n", " return f\n", " return _inner" ] @@ -518,7 +535,7 @@ "metadata": {}, "outputs": [], "source": [ - "mk_class('_t', foo=noops)\n", + "class _t: foo=noops\n", "test_eq(_t().foo(1),1)" ] }, @@ -775,6 +792,18 @@ "assert true([])" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def stop(e=StopIteration):\n", + " \"Raises exception `e` (by default `StopException`)\"\n", + " raise e" + ] + }, { "cell_type": "code", "execution_count": null, @@ -832,7 +861,7 @@ "metadata": {}, "outputs": [], "source": [ - "t = L.range(10)\n", + "t = list(range(10))\n", "test_eq(chunked(t,3), [[0,1,2], [3,4,5], [6,7,8], [9]])\n", "test_eq(chunked(t,3,True), [[0,1,2], [3,4,5], [6,7,8], ])\n", "\n", @@ -842,8 +871,8 @@ "test_eq(chunked(t,3), [[0, 1, 2], [3, 4, 5], [6]])\n", "\n", "t = np.arange(10)\n", - "test_eq(chunked(t,3), L([0,1,2], [3,4,5], [6,7,8], [9]))\n", - "test_eq(chunked(t,3,True), L([0,1,2], [3,4,5], [6,7,8], ))" + "test_eq(chunked(t,3), [[0,1,2], [3,4,5], [6,7,8], [9]])\n", + "test_eq(chunked(t,3,True), [[0,1,2], [3,4,5], [6,7,8], ])" ] }, { @@ -940,68 +969,6 @@ "test_eq(d.b, 3)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def dict2obj(d):\n", - " \"Convert (possibly nested) dicts (or lists of dicts) to `AttrDict`\"\n", - " if isinstance(d, (L,list)): return L(d).map(dict2obj)\n", - " if not isinstance(d, dict): return d\n", - " return AttrDict(**{k:dict2obj(v) for k,v in d.items()})" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This is a convenience to give you \"dotted\" access to (possibly nested) dictionaries, e.g:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'a': 1, 'b': {'c': 2, 'd': 3}}" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "d1 = dict(a=1, b=dict(c=2,d=3))\n", - "d2 = dict2obj(d1)\n", - "test_eq(d2.b.c, 2)\n", - "test_eq(d2.b['c'], 2)\n", - "d2" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It can also be used on lists of dicts." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds = L(d1, d1)\n", - "test_eq(dict2obj(ds)[0].b.c, 2)" - ] - }, { "cell_type": "code", "execution_count": null, @@ -1071,7 +1038,7 @@ " anno = self.__class__.__init__.__annotations__ if cast else {}\n", " if attrs: return _store_attr(self, anno, **attrs)\n", " ns = re.split(', *', names) if names else args[1:]\n", - " _store_attr(self, anno, **{n:fr.f_locals[n] for n in ns if n not in L(but)})" + " _store_attr(self, anno, **{n:fr.f_locals[n] for n in ns if n not in listify(but)})" ] }, { @@ -1191,7 +1158,7 @@ "outputs": [], "source": [ "class T:\n", - " def __init__(self, a:L, b, c:str): store_attr('a,b,c', cast=True)\n", + " def __init__(self, a:listify, b, c:str): store_attr('a,b,c', cast=True)\n", "\n", "t = T(1,c=2,b=3)\n", "assert t.a==[1] and t.b==3 and t.c=='2'" @@ -1610,7 +1577,9 @@ "source": [ "#export\n", "#hide\n", - "class Int(int,ShowPrint): pass" + "class Int(int,ShowPrint):\n", + " \"An extensible `int`\"\n", + " pass" ] }, { @@ -1621,11 +1590,12 @@ "source": [ "#export \n", "#hide\n", - "class Str(str,ShowPrint): pass\n", - "class Float(float,ShowPrint): pass\n", - "add_docs(Str, \"An extensible `str`\");\n", - "add_docs(Int, \"An extensible `int`\");\n", - "add_docs(Float, \"An extensible `float`\")" + "class Str(str,ShowPrint):\n", + " \"An extensible `str`\"\n", + " pass\n", + "class Float(float,ShowPrint):\n", + " \"An extensible `float`\"\n", + " pass" ] }, { @@ -1665,29 +1635,6 @@ "Functions that manipulate popular python collections." ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def tuplify(o, use_list=False, match=None):\n", - " \"Make `o` a tuple\"\n", - " return tuple(L(o, use_list=use_list, match=match))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test_eq(tuplify(None),())\n", - "test_eq(tuplify([1,2,3]),(1,2,3))\n", - "test_eq(tuplify(1,match=[1,2,3]),(1,1,1))" - ] - }, { "cell_type": "code", "execution_count": null, @@ -1735,40 +1682,6 @@ "test_eq(replicate(1, t),(1,1))" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def uniqueify(x, sort=False, bidir=False, start=None):\n", - " \"Unique elements in `x`, optionally `sort`-ed, optionally return reverse correspondence, optionally prepend with elements.\"\n", - " res = L(x).unique()\n", - " if start is not None: res = start+res\n", - " if sort: res.sort()\n", - " if bidir: return res, res.val2idx()\n", - " return res" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# test\n", - "test_eq(set(uniqueify([1,1,0,5,0,3])),{0,1,3,5})\n", - "test_eq(uniqueify([1,1,0,5,0,3], sort=True),[0,1,3,5])\n", - "test_eq(uniqueify([1,1,0,5,0,3], start=[7,8,6]), [7,8,6,1,0,5,3])\n", - "v,o = uniqueify([1,1,0,5,0,3], bidir=True)\n", - "test_eq(v,[1,0,5,3])\n", - "test_eq(o,{1:0, 0: 1, 5: 2, 3: 3})\n", - "v,o = uniqueify([1,1,0,5,0,3], sort=True, bidir=True)\n", - "test_eq(v,[0,1,3,5])\n", - "test_eq(o,{0:0, 1: 1, 3: 2, 5: 3})" - ] - }, { "cell_type": "code", "execution_count": null, @@ -1778,7 +1691,7 @@ "# export\n", "def setify(o): \n", " \"Turn any list like-object into a set.\"\n", - " return o if isinstance(o,set) else set(L(o))" + " return o if isinstance(o,set) else set(listify(o))" ] }, { @@ -1818,31 +1731,6 @@ "test_eq(merge(dict(a=1,b=2), dict(b=3,c=4), None), dict(a=1, b=3, c=4))" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def is_listy(x):\n", - " \"`isinstance(x, (tuple,list,L,slice,Generator))`\"\n", - " return isinstance(x, (tuple,list,L,slice,Generator))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert is_listy((1,))\n", - "assert is_listy([1])\n", - "assert is_listy(L([1]))\n", - "assert is_listy(slice(2))\n", - "assert not is_listy(array([1]))" - ] - }, { "cell_type": "code", "execution_count": null, @@ -1910,32 +1798,6 @@ "test_eq(last_index(6, [1, 2, 9, 3, 4, 9, 10]), -1)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def shufflish(x, pct=0.04):\n", - " \"Randomly relocate items of `x` up to `pct` of `len(x)` from their starting location\"\n", - " n = len(x)\n", - " return L(x[i] for i in sorted(range_of(x), key=lambda o: o+n*(1+random.random()*pct)))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "l = list(range(100))\n", - "l2 = array(shufflish(l))\n", - "test_close(l2[:50 ].mean(), 25, eps=5)\n", - "test_close(l2[-50:].mean(), 75, eps=5)\n", - "test_ne(l,l2)" - ] - }, { "cell_type": "code", "execution_count": null, @@ -1965,7 +1827,7 @@ } ], "source": [ - "letters = L.range(65,73).map_dict(chr)\n", + "letters = {o:chr(o) for o in range(65,73)}\n", "letters" ] }, @@ -1977,7 +1839,7 @@ { "data": { "text/plain": [ - "{70: 'F', 71: 'G'}" + "{65: 'A', 66: 'B', 70: 'F', 71: 'G'}" ] }, "execution_count": null, @@ -1986,7 +1848,7 @@ } ], "source": [ - "filter_dict(letters, lambda k,v: k<3 or v in 'FG')" + "filter_dict(letters, lambda k,v: k<67 or v in 'FG')" ] }, { @@ -2009,7 +1871,7 @@ { "data": { "text/plain": [ - "{}" + "{65: 'A', 66: 'B'}" ] }, "execution_count": null, @@ -2018,7 +1880,7 @@ } ], "source": [ - "filter_keys(letters, lt(3))" + "filter_keys(letters, lt(67))" ] }, { @@ -2054,10 +1916,16 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "## Reindexing Collections" + "#export\n", + "def cycle(o):\n", + " \"Like `itertools.cycle` except creates list of `None`s if `o` is empty\"\n", + " o = listify(o)\n", + " return itertools.cycle(o) if o is not None and len(o) > 0 else itertools.cycle([None])" ] }, { @@ -2066,11 +1934,10 @@ "metadata": {}, "outputs": [], "source": [ - "#export\n", - "#hide\n", - "class IterLen:\n", - " \"Base class to add iteration to anything supporting `__len__` and `__getitem__`\"\n", - " def __iter__(self): return (self[i] for i in range_of(self))" + "test_eq(itertools.islice(cycle([1,2,3]),5), [1,2,3,1,2])\n", + "test_eq(itertools.islice(cycle([]),3), [None]*3)\n", + "test_eq(itertools.islice(cycle(None),3), [None]*3)\n", + "test_eq(itertools.islice(cycle(1),3), [1,1,1])" ] }, { @@ -2080,99 +1947,107 @@ "outputs": [], "source": [ "#export\n", - "@docs\n", - "class ReindexCollection(GetAttr, IterLen):\n", - " \"Reindexes collection `coll` with indices `idxs` and optional LRU cache of size `cache`\"\n", - " _default='coll'\n", - " def __init__(self, coll, idxs=None, cache=None, tfm=noop):\n", - " if idxs is None: idxs = L.range(coll) \n", - " store_attr()\n", - " if cache is not None: self._get = functools.lru_cache(maxsize=cache)(self._get)\n", - "\n", - " def _get(self, i): return self.tfm(self.coll[i])\n", - " def __getitem__(self, i): return self._get(self.idxs[i])\n", - " def __len__(self): return len(self.coll)\n", - " def reindex(self, idxs): self.idxs = idxs\n", - " def shuffle(self): random.shuffle(self.idxs)\n", - " def cache_clear(self): self._get.cache_clear()\n", - " def __getstate__(self): return {'coll': self.coll, 'idxs': self.idxs, 'cache': self.cache, 'tfm': self.tfm}\n", - " def __setstate__(self, s): self.coll,self.idxs,self.cache,self.tfm = s['coll'],s['idxs'],s['cache'],s['tfm']\n", - "\n", - " _docs = dict(reindex=\"Replace `self.idxs` with idxs\",\n", - " shuffle=\"Randomly shuffle indices\",\n", - " cache_clear=\"Clear LRU cache\")" + "def zip_cycle(x, *args):\n", + " \"Like `itertools.zip_longest` but `cycle`s through elements of all but first argument\"\n", + " return zip(x, *map(cycle,args))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "

class ReindexCollection[source]

\n", - "\n", - "> ReindexCollection(**`coll`**, **`idxs`**=*`None`*, **`cache`**=*`None`*, **`tfm`**=*`noop`*) :: [`GetAttr`](/foundation.html#GetAttr)\n", - "\n", - "Reindexes collection `coll` with indices `idxs` and optional LRU cache of size `cache`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "show_doc(ReindexCollection, title_level=4)" + "test_eq(zip_cycle([1,2,3,4],list('abc')), [(1, 'a'), (2, 'b'), (3, 'c'), (4, 'a')])" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "This is useful when constructing batches or organizing data in a particular manner (i.e. for deep learning). This class is primarly used in organizing data for language models in fastai." + "#export\n", + "def sorted_ex(iterable, key=None, reverse=False):\n", + " \"Like `sorted`, but if key is str use `attrgetter`; if int use `itemgetter`\"\n", + " if isinstance(key,str): k=lambda o:getattr(o,key,0)\n", + " elif isinstance(key,int): k=itemgetter(key)\n", + " else: k=key\n", + " return sorted(iterable, key=k, reverse=reverse)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "#### Reindexing\n", - "\n", - "You can supply a custom index upon instantiation with the `idxs` argument, or you can call the `reindex` method to supply a new index for your collection.\n", + "## fastuple\n", "\n", - "Here is how you can reindex a list such that the elements are reversed:" + "A tuple with extended functionality." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['e', 'd', 'c', 'b', 'a']" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "rc=ReindexCollection(['a', 'b', 'c', 'd', 'e'], idxs=[4,3,2,1,0])\n", - "list(rc)" + "#export\n", + "num_methods = \"\"\"\n", + " __add__ __sub__ __mul__ __matmul__ __truediv__ __floordiv__ __mod__ __divmod__ __pow__\n", + " __lshift__ __rshift__ __and__ __xor__ __or__ __neg__ __pos__ __abs__\n", + "\"\"\".split()\n", + "rnum_methods = \"\"\"\n", + " __radd__ __rsub__ __rmul__ __rmatmul__ __rtruediv__ __rfloordiv__ __rmod__ __rdivmod__\n", + " __rpow__ __rlshift__ __rrshift__ __rand__ __rxor__ __ror__\n", + "\"\"\".split()\n", + "inum_methods = \"\"\"\n", + " __iadd__ __isub__ __imul__ __imatmul__ __itruediv__\n", + " __ifloordiv__ __imod__ __ipow__ __ilshift__ __irshift__ __iand__ __ixor__ __ior__\n", + "\"\"\".split()" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "Alternatively, you can use the `reindex` method:" + "#export\n", + "class fastuple(tuple):\n", + " \"A `tuple` with elementwise ops and more friendly __init__ behavior\"\n", + " def __new__(cls, x=None, *rest):\n", + " if x is None: x = ()\n", + " if not isinstance(x,tuple):\n", + " if len(rest): x = (x,)\n", + " else:\n", + " try: x = tuple(iter(x))\n", + " except TypeError: x = (x,)\n", + " return super().__new__(cls, x+rest if rest else x)\n", + "\n", + " def _op(self,op,*args):\n", + " if not isinstance(self,fastuple): self = fastuple(self)\n", + " return type(self)(map(op,self,*map(cycle, args)))\n", + "\n", + " def mul(self,*args):\n", + " \"`*` is already defined in `tuple` for replicating, so use `mul` instead\"\n", + " return fastuple._op(self, operator.mul,*args)\n", + "\n", + " def add(self,*args):\n", + " \"`+` is already defined in `tuple` for concat, so use `add` instead\"\n", + " return fastuple._op(self, operator.add,*args)\n", + "\n", + "def _get_op(op):\n", + " if isinstance(op,str): op = getattr(operator,op)\n", + " def _f(self,*args): return self._op(op,*args)\n", + " return _f\n", + "\n", + "for n in num_methods:\n", + " if not hasattr(fastuple, n) and hasattr(operator,n): setattr(fastuple,n,_get_op(n))\n", + "\n", + "for n in 'eq ne lt le gt ge'.split(): setattr(fastuple,n,_get_op(n))\n", + "setattr(fastuple,'__invert__',_get_op('__not__'))\n", + "setattr(fastuple,'max',_get_op(max))\n", + "setattr(fastuple,'min',_get_op(min))" ] }, { @@ -2183,11 +2058,11 @@ { "data": { "text/markdown": [ - "
ReindexCollection.reindex[source]
\n", + "

class fastuple[source]

\n", "\n", - "> ReindexCollection.reindex(**`idxs`**)\n", + "> fastuple(**`x`**=*`None`*, **\\*`rest`**) :: `tuple`\n", "\n", - "Replace `self.idxs` with idxs" + "A `tuple` with elementwise ops and more friendly __init__ behavior" ], "text/plain": [ "" @@ -2198,127 +2073,53 @@ } ], "source": [ - "show_doc(ReindexCollection.reindex, title_level=6)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['e', 'd', 'c', 'b', 'a']" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "rc=ReindexCollection(['a', 'b', 'c', 'd', 'e'])\n", - "rc.reindex([4,3,2,1,0])\n", - "list(rc)" + "show_doc(fastuple, title_level=4)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "#### LRU Cache" + "#### Friendly init behavior" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "You can optionally specify a LRU cache, which uses [functools.lru_cache](https://docs.python.org/3/library/functools.html#functools.lru_cache) upon instantiation:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "CacheInfo(hits=1, misses=1, maxsize=2, currsize=1)" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sz = 50\n", - "t = ReindexCollection(L.range(sz), cache=2)\n", + "Common failure modes when trying to initialize a tuple in python:\n", "\n", - "#trigger a cache hit by indexing into the same element multiple times\n", - "t[0], t[0]\n", - "t._get.cache_info()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can optionally clear the LRU cache by calling the `cache_clear` method:" + "```py\n", + "tuple(3)\n", + "> TypeError: 'int' object is not iterable\n", + "```\n", + "\n", + "or \n", + "\n", + "```py\n", + "tuple(3, 4)\n", + "> TypeError: tuple expected at most 1 arguments, got 2\n", + "```\n", + "\n", + "However, `fastuple` allows you to define tuples like this and in the usual way:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "
ReindexCollection.cache_clear[source]
\n", - "\n", - "> ReindexCollection.cache_clear()\n", - "\n", - "Clear LRU cache" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "show_doc(ReindexCollection.cache_clear, title_level=5)" + "test_eq(fastuple(3), (3,))\n", + "test_eq(fastuple(3,4), (3, 4))\n", + "test_eq(fastuple((3,4)), (3, 4))" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "CacheInfo(hits=0, misses=0, maxsize=2, currsize=0)" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "sz = 50\n", - "t = ReindexCollection(L.range(sz), cache=2)\n", - "\n", - "#trigger a cache hit by indexing into the same element multiple times\n", - "t[0], t[0]\n", - "t.cache_clear()\n", - "t._get.cache_info()" + "#### Elementwise operations" ] }, { @@ -2329,11 +2130,11 @@ { "data": { "text/markdown": [ - "
ReindexCollection.shuffle[source]
\n", + "
fastuple.add[source]
\n", "\n", - "> ReindexCollection.shuffle()\n", + "> fastuple.add(**\\*`args`**)\n", "\n", - "Randomly shuffle indices" + "`+` is already defined in `tuple` for concat, so use `add` instead" ], "text/plain": [ "" @@ -2344,105 +2145,7 @@ } ], "source": [ - "show_doc(ReindexCollection.shuffle, title_level=5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note that an ordered index is automatically constructed for the data structure even if one is not supplied." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['e', 'd', 'h', 'f', 'c', 'b', 'g', 'a']" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "rc=ReindexCollection(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'])\n", - "rc.shuffle()\n", - "list(rc)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Tests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sz = 50\n", - "t = ReindexCollection(L.range(sz), cache=2)\n", - "test_eq(list(t), range(sz))\n", - "test_eq(t[sz-1], sz-1)\n", - "test_eq(t._get.cache_info().hits, 1)\n", - "t.shuffle()\n", - "test_eq(t._get.cache_info().hits, 1)\n", - "test_ne(list(t), range(sz))\n", - "test_eq(set(t), set(range(sz)))\n", - "t.cache_clear()\n", - "test_eq(t._get.cache_info().hits, 0)\n", - "test_eq(t.count(0), 1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#hide\n", - "#Test ReindexCollection pickles\n", - "t1 = pickle.loads(pickle.dumps(t))\n", - "test_eq(list(t), list(t1))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "num_methods = \"\"\"\n", - " __add__ __sub__ __mul__ __matmul__ __truediv__ __floordiv__ __mod__ __divmod__ __pow__\n", - " __lshift__ __rshift__ __and__ __xor__ __or__ __neg__ __pos__ __abs__\n", - "\"\"\".split()\n", - "rnum_methods = \"\"\"\n", - " __radd__ __rsub__ __rmul__ __rmatmul__ __rtruediv__ __rfloordiv__ __rmod__ __rdivmod__\n", - " __rpow__ __rlshift__ __rrshift__ __rand__ __rxor__ __ror__\n", - "\"\"\".split()\n", - "inum_methods = \"\"\"\n", - " __iadd__ __isub__ __imul__ __imatmul__ __itruediv__\n", - " __ifloordiv__ __imod__ __ipow__ __ilshift__ __irshift__ __iand__ __ixor__ __ior__\n", - "\"\"\".split()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## fastuple\n", - "\n", - "A tuple with extended functionality." + "show_doc(fastuple.add, title_level=5)" ] }, { @@ -2451,42 +2154,9 @@ "metadata": {}, "outputs": [], "source": [ - "#export\n", - "class fastuple(tuple):\n", - " \"A `tuple` with elementwise ops and more friendly __init__ behavior\"\n", - " def __new__(cls, x=None, *rest):\n", - " if x is None: x = ()\n", - " if not isinstance(x,tuple):\n", - " if len(rest): x = (x,)\n", - " else:\n", - " try: x = tuple(iter(x))\n", - " except TypeError: x = (x,)\n", - " return super().__new__(cls, x+rest if rest else x)\n", - "\n", - " def _op(self,op,*args):\n", - " if not isinstance(self,fastuple): self = fastuple(self)\n", - " return type(self)(map(op,self,*map(cycle, args)))\n", - "\n", - " def mul(self,*args):\n", - " \"`*` is already defined in `tuple` for replicating, so use `mul` instead\"\n", - " return fastuple._op(self, operator.mul,*args)\n", - "\n", - " def add(self,*args):\n", - " \"`+` is already defined in `tuple` for concat, so use `add` instead\"\n", - " return fastuple._op(self, operator.add,*args)\n", - "\n", - "def _get_op(op):\n", - " if isinstance(op,str): op = getattr(operator,op)\n", - " def _f(self,*args): return self._op(op,*args)\n", - " return _f\n", - "\n", - "for n in num_methods:\n", - " if not hasattr(fastuple, n) and hasattr(operator,n): setattr(fastuple,n,_get_op(n))\n", - "\n", - "for n in 'eq ne lt le gt ge'.split(): setattr(fastuple,n,_get_op(n))\n", - "setattr(fastuple,'__invert__',_get_op('__not__'))\n", - "setattr(fastuple,'max',_get_op(max))\n", - "setattr(fastuple,'min',_get_op(min))" + "test_eq(fastuple.add((1,1),(2,2)), (3,3))\n", + "test_eq_type(fastuple(1,1).add(2), fastuple(3,3))\n", + "test_eq(fastuple('1','2').add('2'), fastuple('12','22'))" ] }, { @@ -2497,120 +2167,11 @@ { "data": { "text/markdown": [ - "

class fastuple[source]

\n", + "
fastuple.mul[source]
\n", "\n", - "> fastuple(**`x`**=*`None`*, **\\*`rest`**) :: `tuple`\n", + "> fastuple.mul(**\\*`args`**)\n", "\n", - "A `tuple` with elementwise ops and more friendly __init__ behavior" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "show_doc(fastuple, title_level=4)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Friendly init behavior" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Common failure modes when trying to initialize a tuple in python:\n", - "\n", - "```py\n", - "tuple(3)\n", - "> TypeError: 'int' object is not iterable\n", - "```\n", - "\n", - "or \n", - "\n", - "```py\n", - "tuple(3, 4)\n", - "> TypeError: tuple expected at most 1 arguments, got 2\n", - "```\n", - "\n", - "However, `fastuple` allows you to define tuples like this and in the usual way:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test_eq(fastuple(3), (3,))\n", - "test_eq(fastuple(3,4), (3, 4))\n", - "test_eq(fastuple((3,4)), (3, 4))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Elementwise operations" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "
fastuple.add[source]
\n", - "\n", - "> fastuple.add(**\\*`args`**)\n", - "\n", - "`+` is already defined in `tuple` for concat, so use `add` instead" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "show_doc(fastuple.add, title_level=5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test_eq(fastuple.add((1,1),(2,2)), (3,3))\n", - "test_eq_type(fastuple(1,1).add(2), fastuple(3,3))\n", - "test_eq(fastuple('1','2').add('2'), fastuple('12','22'))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "
fastuple.mul[source]
\n", - "\n", - "> fastuple.mul(**\\*`args`**)\n", - "\n", - "`*` is already defined in `tuple` for replicating, so use `mul` instead" + "`*` is already defined in `tuple` for replicating, so use `mul` instead" ], "text/plain": [ "" @@ -2630,1044 +2191,20 @@ "metadata": {}, "outputs": [], "source": [ - "test_eq_type(fastuple(1,1).mul(2), fastuple(2,2))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Other Elementwise Operations\n", - "\n", - "Additionally, the following elementwise operations are available:\n", - "- `le`: less than\n", - "- `eq`: equal\n", - "- `gt`: greater than\n", - "- `min`: minimum of" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test_eq(fastuple(3,1).le(1), (False, True))\n", - "test_eq(fastuple(3,1).eq(1), (False, True))\n", - "test_eq(fastuple(3,1).gt(1), (True, False))\n", - "test_eq(fastuple(3,1).min(2), (2,1))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can also do other elemntwise operations like negate a `fastuple`, or subtract two `fastuple`s:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test_eq(-fastuple(1,2), (-1,-2))\n", - "test_eq(~fastuple(1,0,1), (False,True,False))\n", - "\n", - "test_eq(fastuple(1,1)-fastuple(2,2), (-1,-1))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Other Tests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test_eq(type(fastuple(1)), fastuple)\n", - "test_eq_type(fastuple(1,2), fastuple(1,2))\n", - "test_ne(fastuple(1,2), fastuple(1,3))\n", - "test_eq(fastuple(), ())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Functions on Functions\n", - "\n", - "Utilities for functional programming or for defining, modifying, or debugging functions. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def trace(f):\n", - " \"Add `set_trace` to an existing function `f`\"\n", - " if getattr(f, '_traced', False): return f\n", - " def _inner(*args,**kwargs):\n", - " set_trace()\n", - " return f(*args,**kwargs)\n", - " _inner._traced = True\n", - " return _inner" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can add a breakpoint to an existing function, e.g:\n", - "\n", - "```python\n", - "Path.cwd = trace(Path.cwd)\n", - "Path.cwd()\n", - "```\n", - "\n", - "Now, when the function is called it will drop you into the debugger. Note, you must issue the `s` command when you begin to step into the function that is being traced." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "def compose(*funcs, order=None):\n", - " \"Create a function that composes all functions in `funcs`, passing along remaining `*args` and `**kwargs` to all\"\n", - " funcs = L(funcs)\n", - " if len(funcs)==0: return noop\n", - " if len(funcs)==1: return funcs[0]\n", - " if order is not None: funcs = funcs.sorted(order)\n", - " def _inner(x, *args, **kwargs):\n", - " for f in L(funcs): x = f(x, *args, **kwargs)\n", - " return x\n", - " return _inner" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "f1 = lambda o,p=0: (o*2)+p\n", - "f2 = lambda o,p=1: (o+1)/p\n", - "test_eq(f2(f1(3)), compose(f1,f2)(3))\n", - "test_eq(f2(f1(3,p=3),p=3), compose(f1,f2)(3,p=3))\n", - "test_eq(f2(f1(3, 3), 3), compose(f1,f2)(3, 3))\n", - "\n", - "f1.order = 1\n", - "test_eq(f1(f2(3)), compose(f1,f2, order=\"order\")(3))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def maps(*args, retain=noop):\n", - " \"Like `map`, except funcs are composed first\"\n", - " f = compose(*args[:-1])\n", - " def _f(b): return retain(f(b), b)\n", - " return map(_f, args[-1])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test_eq(maps([1]), [1])\n", - "test_eq(maps(operator.neg, [1,2]), [-1,-2])\n", - "test_eq(maps(operator.neg, operator.neg, [1,2]), [1,2])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def partialler(f, *args, order=None, **kwargs):\n", - " \"Like `functools.partial` but also copies over docstring\"\n", - " fnew = partial(f,*args,**kwargs)\n", - " fnew.__doc__ = f.__doc__\n", - " if order is not None: fnew.order=order\n", - " elif hasattr(f,'order'): fnew.order=f.order\n", - " return fnew" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def _f(x,a=1):\n", - " \"test func\"\n", - " return x-a\n", - "_f.order=1\n", - "\n", - "f = partialler(_f, 2)\n", - "test_eq(f.order, 1)\n", - "test_eq(f(3), -1)\n", - "f = partialler(_f, a=2, order=3)\n", - "test_eq(f.__doc__, \"test func\")\n", - "test_eq(f.order, 3)\n", - "test_eq(f(3), _f(3,2))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class partial0:\n", - " \"Like `partialler`, but args passed to callable are inserted at started, instead of at end\"\n", - " def __init__(self, f, *args, order=None, **kwargs):\n", - " self.f,self.args,self.kwargs = f,args,kwargs\n", - " self.order = ifnone(order, getattr(f,'order',None))\n", - " self.__doc__ = f.__doc__\n", - "\n", - " def __call__(self, *args, **kwargs): return self.f(*args, *self.args, **kwargs, **self.kwargs)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "f = partial0(_f, 2)\n", - "test_eq(f.order, 1)\n", - "test_eq(f(3), 1) # NB: different to `partialler` example" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def mapped(f, it):\n", - " \"map `f` over `it`, unless it's not listy, in which case return `f(it)`\"\n", - " return L(it).map(f) if is_listy(it) else f(it)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test_eq(mapped(_f,1),0)\n", - "test_eq(mapped(_f,[1,2]),[0,1])\n", - "test_eq(mapped(_f,(1,)),(0,))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def instantiate(t):\n", - " \"Instantiate `t` if it's a type, otherwise do nothing\"\n", - " return t() if isinstance(t, type) else t" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test_eq_type(instantiate(int), 0)\n", - "test_eq_type(instantiate(1), 1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def _using_attr(f, attr, x): return f(getattr(x,attr))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def using_attr(f, attr):\n", - " \"Change function `f` to operate on `attr`\"\n", - " return partial(_using_attr, f, attr)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "t = Path('/a/b.txt')\n", - "f = using_attr(str.upper, 'name')\n", - "test_eq(f(t), 'B.TXT')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Self (with an _uppercase_ S)\n", - "\n", - "A Concise Way To Create Lambdas" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "class _Self:\n", - " \"An alternative to `lambda` for calling methods on passed object.\"\n", - " def __init__(self): self.nms,self.args,self.kwargs,self.ready = [],[],[],True\n", - " def __repr__(self): return f'self: {self.nms}({self.args}, {self.kwargs})'\n", - "\n", - " def __call__(self, *args, **kwargs):\n", - " if self.ready:\n", - " x = args[0]\n", - " for n,a,k in zip(self.nms,self.args,self.kwargs):\n", - " x = getattr(x,n)\n", - " if callable(x) and a is not None: x = x(*a, **k)\n", - " return x\n", - " else:\n", - " self.args.append(args)\n", - " self.kwargs.append(kwargs)\n", - " self.ready = True\n", - " return self\n", - "\n", - " def __getattr__(self,k):\n", - " if not self.ready:\n", - " self.args.append(None)\n", - " self.kwargs.append(None)\n", - " self.nms.append(k)\n", - " self.ready = False\n", - " return self" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "class _SelfCls:\n", - " def __getattr__(self,k): return getattr(_Self(),k)\n", - " def __getitem__(self,i): return self.__getattr__('__getitem__')(i)\n", - "\n", - "Self = _SelfCls()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "_all_ = ['Self']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This is a concise way to create lambdas that are calling methods on an object (note the capitalization!)\n", - "\n", - "`Self.sum()`, for instance, is a shortcut for `lambda o: o.sum()`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "f = Self.sum()\n", - "x = array([3.,1])\n", - "test_eq(f(x), 4.)\n", - "\n", - "# This is equivalent to above\n", - "f = lambda o: o.sum()\n", - "x = array([3.,1])\n", - "test_eq(f(x), 4.)\n", - "\n", - "f = Self.argmin()\n", - "arr = np.array([1,2,3,4,5])\n", - "test_eq(f(arr), arr.argmin())\n", - "\n", - "f = Self.sum().is_integer()\n", - "x = array([3.,1])\n", - "test_eq(f(x), True)\n", - "\n", - "f = Self.sum().real.is_integer()\n", - "x = array([3.,1])\n", - "test_eq(f(x), True)\n", - "\n", - "f = Self.imag()\n", - "test_eq(f(3), 0)\n", - "\n", - "f = Self[1]\n", - "test_eq(f(x), 1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Extensions to Pathlib.Path" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "An extension of the standard python libary [Pathlib.Path](https://docs.python.org/3/library/pathlib.html#basic-use). These extensions are accomplished by monkey patching additional methods onto `Pathlib.Path`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "@patch\n", - "def readlines(self:Path, hint=-1, encoding='utf8'):\n", - " \"Read the content of `self`\"\n", - " with self.open(encoding=encoding) as f: return f.readlines(hint)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "@patch\n", - "def mk_write(self:Path, data, encoding=None, errors=None, mode=511):\n", - " \"Make all parent dirs of `self`\"\n", - " self.parent.mkdir(exist_ok=True, parents=True, mode=mode)\n", - " self.write_text(data, encoding=encoding, errors=errors)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "@patch\n", - "def ls(self:Path, n_max=None, file_type=None, file_exts=None):\n", - " \"Contents of path as a list\"\n", - " extns=L(file_exts)\n", - " if file_type: extns += L(k for k,v in mimetypes.types_map.items() if v.startswith(file_type+'/'))\n", - " has_extns = len(extns)==0\n", - " res = (o for o in self.iterdir() if has_extns or o.suffix in extns)\n", - " if n_max is not None: res = itertools.islice(res, n_max)\n", - " return L(res)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We add an `ls()` method to `pathlib.Path` which is simply defined as `list(Path.iterdir())`, mainly for convenience in REPL environments such as notebooks." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Path('.ipynb_checkpoints')" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "path = Path()\n", - "t = path.ls()\n", - "assert len(t)>0\n", - "t1 = path.ls(10)\n", - "test_eq(len(t1), 10)\n", - "t2 = path.ls(file_exts='.ipynb')\n", - "assert len(t)>len(t2)\n", - "t[0]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can also pass an optional `file_type` MIME prefix and/or a list of file extensions." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(Path('../fastcore/logargs.py'), Path('04_transform.ipynb'))" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "lib_path = (path/'../fastcore')\n", - "txt_files=lib_path.ls(file_type='text')\n", - "assert len(txt_files) > 0 and txt_files[0].suffix=='.py'\n", - "ipy_files=path.ls(file_exts=['.ipynb'])\n", - "assert len(ipy_files) > 0 and ipy_files[0].suffix=='.ipynb'\n", - "txt_files[0],ipy_files[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#hide\n", - "path = Path()\n", - "pkl = pickle.dumps(path)\n", - "p2 = pickle.loads(pkl)\n", - "test_eq(path.ls()[0], p2.ls()[0])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def open_file(fn, mode='r'):\n", - " \"Open a file, with optional compression if gz or bz2 suffix\"\n", - " if isinstance(fn, io.IOBase): return fn\n", - " fn = Path(fn)\n", - " if fn.suffix=='.bz2': return bz2.BZ2File(fn, mode)\n", - " elif fn.suffix=='.gz' : return gzip.GzipFile(fn, mode)\n", - " else: return open(fn,mode)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def save_pickle(fn, o):\n", - " \"Save a pickle file, to a file name or opened file\"\n", - " with open_file(fn, 'wb') as f: pickle.dump(o, f)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def load_pickle(fn):\n", - " \"Load a pickle file from a file name or opened file\"\n", - " with open_file(fn, 'rb') as f: return pickle.load(f)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for suf in '.pkl','.bz2','.gz':\n", - " with tempfile.NamedTemporaryFile(suffix=suf) as f:\n", - " fn = Path(f.name)\n", - " save_pickle(fn, 't')\n", - " t = load_pickle(fn)\n", - " test_eq(t,'t')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "@patch\n", - "def __repr__(self:Path):\n", - " b = getattr(Path, 'BASE_PATH', None)\n", - " if b:\n", - " try: self = self.relative_to(b)\n", - " except: pass\n", - " return f\"Path({self.as_posix()!r})\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "fastai also updates the `repr` of `Path` such that, if `Path.BASE_PATH` is defined, all paths are printed relative to that path (as long as they are contained in `Path.BASE_PATH`:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "t = ipy_files[0].absolute()\n", - "try:\n", - " Path.BASE_PATH = t.parent.parent\n", - " test_eq(repr(t), f\"Path('nbs/{t.name}')\")\n", - "finally: Path.BASE_PATH = None" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## File Functions\n", - "\n", - "Utilities (other than extensions to Pathlib.Path) for dealing with IO." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def bunzip(fn):\n", - " \"bunzip `fn`, raising exception if output already exists\"\n", - " fn = Path(fn)\n", - " assert fn.exists(), f\"{fn} doesn't exist\"\n", - " out_fn = fn.with_suffix('')\n", - " assert not out_fn.exists(), f\"{out_fn} already exists\"\n", - " with bz2.BZ2File(fn, 'rb') as src, out_fn.open('wb') as dst:\n", - " for d in iter(lambda: src.read(1024*1024), b''): dst.write(d)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "f = Path('files/test.txt')\n", - "if f.exists(): f.unlink()\n", - "bunzip('files/test.txt.bz2')\n", - "t = f.open().readlines()\n", - "test_eq(len(t),1)\n", - "test_eq(t[0], 'test\\n')\n", - "f.unlink()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def join_path_file(file, path, ext=''):\n", - " \"Return `path/file` if file is a string or a `Path`, file otherwise\"\n", - " if not isinstance(file, (str, Path)): return file\n", - " path.mkdir(parents=True, exist_ok=True)\n", - " return path/f'{file}{ext}'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "path = Path.cwd()/'_tmp'/'tst'\n", - "f = join_path_file('tst.txt', path)\n", - "assert path.exists()\n", - "test_eq(f, path/'tst.txt')\n", - "with open(f, 'w') as f_: assert join_path_file(f_, path) == f_\n", - "shutil.rmtree(Path.cwd()/'_tmp')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def urlread(url, data=None, **kwargs):\n", - " \"Retrieve `url`, using `data` dict or `kwargs` to `POST` if present\"\n", - " if kwargs and not data: data=kwargs\n", - " if data is not None:\n", - " if not isinstance(data, (str,bytes)): data = urlencode(data)\n", - " if not isinstance(data, bytes): data = data.encode('ascii')\n", - " cls = urllib.request.Request\n", - " if not isinstance(url,cls): url = cls(url)\n", - " url.headers['User-Agent'] = 'Mozilla/5.0'\n", - " with urlopen(url, data=data) as res: return res.read()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def urljson(url, data=None):\n", - " \"Retrieve `url` and decode json\"\n", - " return json.loads(urlread(url, data=data))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def run(cmd, *rest, ignore_ex=False, as_bytes=False):\n", - " \"Pass `cmd` (splitting with `shlex` if string) to `subprocess.run`; return `stdout`; raise `IOError` if fails\"\n", - " if rest: cmd = (cmd,)+rest\n", - " elif isinstance(cmd,str): cmd = shlex.split(cmd)\n", - " res = subprocess.run(cmd, capture_output=True)\n", - " stdout = res.stdout\n", - " if not as_bytes: stdout = stdout.decode()\n", - " if ignore_ex: return (res.returncode, stdout)\n", - " if res.returncode: raise IOError(\"{} ;; {}\".format(res.stdout, res.stderr))\n", - " return stdout" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can pass a string (which will be split based on standard shell rules), a list, or pass args directly:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert 'ipynb' in run('ls -l')\n", - "assert 'ipynb' in run(['ls', '-l'])\n", - "assert 'ipynb' in run('ls', '-l')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Some commands fail in non-error situations, like `grep`. Use `ignore_ex` in those cases, which will return a tuple of stdout and returncode:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test_eq(run('grep asdfds 00_test.ipynb', ignore_ex=True)[0], 1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "`run` automatically decodes returned bytes to a `str`. Use `as_bytes` to skip that:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test_eq(run('echo hi', as_bytes=True), b'hi\\n')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def do_request(url, post=False, headers=None, **data):\n", - " \"Call GET or json-encoded POST on `url`, depending on `post`\"\n", - " if data:\n", - " if post: data = json.dumps(data).encode('ascii')\n", - " else:\n", - " url += \"?\" + urlencode(data)\n", - " data = None\n", - " return urljson(Request(url, headers=headers, data=data or None))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Sorting Objects From Before/After" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Transforms and callbacks will have run_after/run_before attributes, this function will sort them to respect those requirements (if it's possible). Also, sometimes we want a tranform/callback to be run at the end, but still be able to use run_after/run_before behaviors. For those, the function checks for a toward_end attribute (that needs to be True)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def _is_instance(f, gs):\n", - " tst = [g if type(g) in [type, 'function'] else g.__class__ for g in gs]\n", - " for g in tst:\n", - " if isinstance(f, g) or f==g: return True\n", - " return False\n", - "\n", - "def _is_first(f, gs):\n", - " for o in L(getattr(f, 'run_after', None)):\n", - " if _is_instance(o, gs): return False\n", - " for g in gs:\n", - " if _is_instance(f, L(getattr(g, 'run_before', None))): return False\n", - " return True\n", - "\n", - "def sort_by_run(fs):\n", - " end = L(fs).attrgot('toward_end')\n", - " inp,res = L(fs)[~end] + L(fs)[end], L()\n", - " while len(inp):\n", - " for i,o in enumerate(inp):\n", - " if _is_first(o, inp):\n", - " res.append(inp.pop(i))\n", - " break\n", - " else: raise Exception(\"Impossible to sort\")\n", - " return res" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class Tst(): pass \n", - "class Tst1():\n", - " run_before=[Tst]\n", - "class Tst2():\n", - " run_before=Tst\n", - " run_after=Tst1\n", - " \n", - "tsts = [Tst(), Tst1(), Tst2()]\n", - "test_eq(sort_by_run(tsts), [tsts[1], tsts[2], tsts[0]])\n", - "\n", - "Tst2.run_before,Tst2.run_after = Tst1,Tst\n", - "test_fail(lambda: sort_by_run([Tst(), Tst1(), Tst2()]))\n", - "\n", - "def tst1(x): return x\n", - "tst1.run_before = Tst\n", - "test_eq(sort_by_run([tsts[0], tst1]), [tst1, tsts[0]])\n", - " \n", - "class Tst1():\n", - " toward_end=True\n", - "class Tst2():\n", - " toward_end=True\n", - " run_before=Tst1\n", - "tsts = [Tst(), Tst1(), Tst2()]\n", - "test_eq(sort_by_run(tsts), [tsts[0], tsts[2], tsts[1]])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Other Helpers" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "class PrettyString(str):\n", - " \"Little hack to get strings to show properly in Jupyter.\"\n", - " def __repr__(self): return self" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "

class PrettyString[source]

\n", - "\n", - "> PrettyString() :: `str`\n", - "\n", - "Little hack to get strings to show properly in Jupyter." - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "show_doc(PrettyString, title_level=4)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Allow strings with special characters to render properly in Jupyter. Without calling `print()` strings with special characters are displayed like so:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'a string\\nwith\\nnew\\nlines and\\ttabs'" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "with_special_chars='a string\\nwith\\nnew\\nlines and\\ttabs'\n", - "with_special_chars" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can correct this with `PrettyString`:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "a string\n", - "with\n", - "new\n", - "lines and\ttabs" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "PrettyString(with_special_chars)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def round_multiple(x, mult, round_down=False):\n", - " \"Round `x` to nearest multiple of `mult`\"\n", - " def _f(x_): return (int if round_down else round)(x_/mult)*mult\n", - " res = L(x).map(_f)\n", - " return res if is_listy(x) else res[0]" + "test_eq_type(fastuple(1,1).mul(2), fastuple(2,2))" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "test_eq(round_multiple(63,32), 64)\n", - "test_eq(round_multiple(50,32), 64)\n", - "test_eq(round_multiple(40,32), 32)\n", - "test_eq(round_multiple( 0,32), 0)\n", - "test_eq(round_multiple(63,32, round_down=True), 32)\n", - "test_eq(round_multiple((63,40),32), (64,32))" + "#### Other Elementwise Operations\n", + "\n", + "Additionally, the following elementwise operations are available:\n", + "- `le`: less than\n", + "- `eq`: equal\n", + "- `gt`: greater than\n", + "- `min`: minimum of" ] }, { @@ -3676,24 +2213,17 @@ "metadata": {}, "outputs": [], "source": [ - "#export\n", - "def even_mults(start, stop, n):\n", - " \"Build log-stepped array from `start` to `stop` in `n` steps.\"\n", - " if n==1: return stop\n", - " mult = stop/start\n", - " step = mult**(1/(n-1))\n", - " return [start*(step**i) for i in range(n)]" + "test_eq(fastuple(3,1).le(1), (False, True))\n", + "test_eq(fastuple(3,1).eq(1), (False, True))\n", + "test_eq(fastuple(3,1).gt(1), (True, False))\n", + "test_eq(fastuple(3,1).min(2), (2,1))" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "test_eq(even_mults(2,8,3), [2,4,8])\n", - "test_eq(even_mults(2,32,5), [2,4,8,16,32])\n", - "test_eq(even_mults(2,8,1), 8)" + "You can also do other elemntwise operations like negate a `fastuple`, or subtract two `fastuple`s:" ] }, { @@ -3702,33 +2232,10 @@ "metadata": {}, "outputs": [], "source": [ - "#export\n", - "def num_cpus():\n", - " \"Get number of cpus\"\n", - " try: return len(os.sched_getaffinity(0))\n", - " except AttributeError: return os.cpu_count()\n", + "test_eq(-fastuple(1,2), (-1,-2))\n", + "test_eq(~fastuple(1,0,1), (False,True,False))\n", "\n", - "defaults.cpus = num_cpus()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "64" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "num_cpus()" + "test_eq(fastuple(1,1)-fastuple(2,2), (-1,-1))" ] }, { @@ -3737,24 +2244,19 @@ "metadata": {}, "outputs": [], "source": [ - "#export\n", - "def add_props(f, g=None, n=2):\n", - " \"Create properties passing each of `range(n)` to f\"\n", - " if g is None: return (property(partial(f,i)) for i in range(n))\n", - " return (property(partial(f,i), partial(g,i)) for i in range(n))" + "test_eq(type(fastuple(1)), fastuple)\n", + "test_eq_type(fastuple(1,2), fastuple(1,2))\n", + "test_ne(fastuple(1,2), fastuple(1,3))\n", + "test_eq(fastuple(), ())" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "class _T(): a,b = add_props(lambda i,x:i*2)\n", + "## Functions on Functions\n", "\n", - "t = _T()\n", - "test_eq(t.a,0)\n", - "test_eq(t.b,2)" + "Utilities for functional programming or for defining, modifying, or debugging functions. " ] }, { @@ -3763,18 +2265,17 @@ "metadata": {}, "outputs": [], "source": [ - "class _T(): \n", - " def __init__(self, v): self.v=v\n", - " def _set(i, self, v): self.v[i] = v\n", - " a,b = add_props(lambda i,x: x.v[i], _set)\n", - "\n", - "t = _T([0,2])\n", - "test_eq(t.a,0)\n", - "test_eq(t.b,2)\n", - "t.a = t.a+1\n", - "t.b = 3\n", - "test_eq(t.a,1)\n", - "test_eq(t.b,3)" + "# export\n", + "def compose(*funcs, order=None):\n", + " \"Create a function that composes all functions in `funcs`, passing along remaining `*args` and `**kwargs` to all\"\n", + " funcs = listify(funcs)\n", + " if len(funcs)==0: return noop\n", + " if len(funcs)==1: return funcs[0]\n", + " if order is not None: funcs = sorted_ex(funcs, key=order)\n", + " def _inner(x, *args, **kwargs):\n", + " for f in funcs: x = f(x, *args, **kwargs)\n", + " return x\n", + " return _inner" ] }, { @@ -3783,8 +2284,14 @@ "metadata": {}, "outputs": [], "source": [ - "# export\n", - "from contextlib import ExitStack" + "f1 = lambda o,p=0: (o*2)+p\n", + "f2 = lambda o,p=1: (o+1)/p\n", + "test_eq(f2(f1(3)), compose(f1,f2)(3))\n", + "test_eq(f2(f1(3,p=3),p=3), compose(f1,f2)(3,p=3))\n", + "test_eq(f2(f1(3, 3), 3), compose(f1,f2)(3, 3))\n", + "\n", + "f1.order = 1\n", + "test_eq(f1(f2(3)), compose(f1,f2, order=\"order\")(3))" ] }, { @@ -3794,37 +2301,22 @@ "outputs": [], "source": [ "#export\n", - "class ContextManagers(GetAttr):\n", - " \"Wrapper for `contextlib.ExitStack` which enters a collection of context managers\"\n", - " def __init__(self, mgrs): self.default,self.stack = L(mgrs),ExitStack()\n", - " def __enter__(self): self.default.map(self.stack.enter_context)\n", - " def __exit__(self, *args, **kwargs): self.stack.__exit__(*args, **kwargs)" + "def maps(*args, retain=noop):\n", + " \"Like `map`, except funcs are composed first\"\n", + " f = compose(*args[:-1])\n", + " def _f(b): return retain(f(b), b)\n", + " return map(_f, args[-1])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "

class ContextManagers[source]

\n", - "\n", - "> ContextManagers(**`mgrs`**) :: [`GetAttr`](/foundation.html#GetAttr)\n", - "\n", - "Wrapper for `contextlib.ExitStack` which enters a collection of context managers" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "show_doc(ContextManagers, title_level=4)" + "test_eq(maps([1]), [1])\n", + "test_eq(maps(operator.neg, [1,2]), [-1,-2])\n", + "test_eq(maps(operator.neg, operator.neg, [1,2]), [1,2])" ] }, { @@ -3834,7 +2326,13 @@ "outputs": [], "source": [ "#export\n", - "def _typeerr(arg, val, typ): return TypeError(f\"{arg}=={val} not {typ}\")" + "def partialler(f, *args, order=None, **kwargs):\n", + " \"Like `functools.partial` but also copies over docstring\"\n", + " fnew = partial(f,*args,**kwargs)\n", + " fnew.__doc__ = f.__doc__\n", + " if order is not None: fnew.order=order\n", + " elif hasattr(f,'order'): fnew.order=f.order\n", + " return fnew" ] }, { @@ -3843,31 +2341,34 @@ "metadata": {}, "outputs": [], "source": [ - "#export\n", - "def typed(f):\n", - " \"Decorator to check param and return types at runtime\"\n", - " names = f.__code__.co_varnames\n", - " anno = f.__annotations__\n", - " ret = anno.pop('return',None)\n", - " def _f(*args,**kwargs):\n", - " kw = {**kwargs}\n", - " if len(anno) > 0:\n", - " for i,arg in enumerate(args): kw[names[i]] = arg\n", - " for k,v in kw.items():\n", - " if not isinstance(v,anno[k]): raise _typeerr(k, v, anno[k])\n", - " res = f(*args,**kwargs)\n", - " if ret is not None and not isinstance(res,ret): raise _typeerr(\"return\", res, ret)\n", - " return res\n", - " return functools.update_wrapper(_f, f)" + "def _f(x,a=1):\n", + " \"test func\"\n", + " return x-a\n", + "_f.order=1\n", + "\n", + "f = partialler(_f, 2)\n", + "test_eq(f.order, 1)\n", + "test_eq(f(3), -1)\n", + "f = partialler(_f, a=2, order=3)\n", + "test_eq(f.__doc__, \"test func\")\n", + "test_eq(f.order, 3)\n", + "test_eq(f(3), _f(3,2))" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "`typed` validates argument types at **runtime**. This is in contrast to [MyPy](http://mypy-lang.org/) which only offers static type checking.\n", + "class partial0:\n", + " \"Like `partialler`, but args passed to callable are inserted at started, instead of at end\"\n", + " def __init__(self, f, *args, order=None, **kwargs):\n", + " self.f,self.args,self.kwargs = f,args,kwargs\n", + " self.order = ifnone(order, getattr(f,'order',None))\n", + " self.__doc__ = f.__doc__\n", "\n", - "For example, a `TypeError` will be raised if we try to pass an integer into the first argument of the below function: " + " def __call__(self, *args, **kwargs): return self.f(*args, *self.args, **kwargs, **self.kwargs)" ] }, { @@ -3876,18 +2377,21 @@ "metadata": {}, "outputs": [], "source": [ - "@typed\n", - "def discount(price:int, pct:float): \n", - " return (1-pct) * price\n", - "\n", - "with ExceptionExpected(TypeError): discount(100.0, .1)" + "f = partial0(_f, 2)\n", + "test_eq(f.order, 1)\n", + "test_eq(f(3), 1) # NB: different to `partialler` example" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "We can also optionally allow multiple types by enumarating the types in a tuple as illustrated below:" + "#export\n", + "def instantiate(t):\n", + " \"Instantiate `t` if it's a type, otherwise do nothing\"\n", + " return t() if isinstance(t, type) else t" ] }, { @@ -3896,10 +2400,8 @@ "metadata": {}, "outputs": [], "source": [ - "def discount(price:(int,float), pct:float): \n", - " return (1-pct) * price\n", - "\n", - "assert 90.0 == discount(100.0, .1)" + "test_eq_type(instantiate(int), 0)\n", + "test_eq_type(instantiate(1), 1)" ] }, { @@ -3908,19 +2410,8 @@ "metadata": {}, "outputs": [], "source": [ - "@typed\n", - "def foo(a:int, b:str='a'): return a\n", - "test_eq(foo(1, '2'), 1)\n", - "\n", - "with ExceptionExpected(TypeError): foo(1,2)\n", - "\n", - "@typed\n", - "def foo()->str: return 1\n", - "with ExceptionExpected(TypeError): foo()\n", - "\n", - "@typed\n", - "def foo()->str: return '1'\n", - "assert foo()" + "#export\n", + "def _using_attr(f, attr, x): return f(getattr(x,attr))" ] }, { @@ -3930,10 +2421,9 @@ "outputs": [], "source": [ "#export\n", - "def str2bool(s):\n", - " \"Case-insensitive convert string `s` too a bool (`y`,`yes`,`t`,`true`,`on`,`1`->`True`)\"\n", - " if not isinstance(s,str): return bool(s)\n", - " return bool(distutils.util.strtobool(s)) if s else False" + "def using_attr(f, attr):\n", + " \"Change function `f` to operate on `attr`\"\n", + " return partial(_using_attr, f, attr)" ] }, { @@ -3942,17 +2432,52 @@ "metadata": {}, "outputs": [], "source": [ - "for o in \"y YES t True on 1\".split(): assert str2bool(o)\n", - "for o in \"n no FALSE off 0\".split(): assert not str2bool(o)\n", - "for o in 0,None,'',False: assert not str2bool(o)\n", - "for o in 1,True: assert str2bool(o)" + "t = Path('/a/b.txt')\n", + "f = using_attr(str.upper, 'name')\n", + "test_eq(f(t), 'B.TXT')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Multiprocessing" + "### Self (with an _uppercase_ S)\n", + "\n", + "A Concise Way To Create Lambdas" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "class _Self:\n", + " \"An alternative to `lambda` for calling methods on passed object.\"\n", + " def __init__(self): self.nms,self.args,self.kwargs,self.ready = [],[],[],True\n", + " def __repr__(self): return f'self: {self.nms}({self.args}, {self.kwargs})'\n", + "\n", + " def __call__(self, *args, **kwargs):\n", + " if self.ready:\n", + " x = args[0]\n", + " for n,a,k in zip(self.nms,self.args,self.kwargs):\n", + " x = getattr(x,n)\n", + " if callable(x) and a is not None: x = x(*a, **k)\n", + " return x\n", + " else:\n", + " self.args.append(args)\n", + " self.kwargs.append(kwargs)\n", + " self.ready = True\n", + " return self\n", + "\n", + " def __getattr__(self,k):\n", + " if not self.ready:\n", + " self.args.append(None)\n", + " self.kwargs.append(None)\n", + " self.nms.append(k)\n", + " self.ready = False\n", + " return self" ] }, { @@ -3962,10 +2487,11 @@ "outputs": [], "source": [ "#export\n", - "from multiprocessing import Process, Queue\n", - "import concurrent.futures\n", - "import time\n", - "from multiprocessing import Manager" + "class _SelfCls:\n", + " def __getattr__(self,k): return getattr(_Self(),k)\n", + " def __getitem__(self,i): return self.__getattr__('__getitem__')(i)\n", + "\n", + "Self = _SelfCls()" ] }, { @@ -3975,25 +2501,16 @@ "outputs": [], "source": [ "#export\n", - "def set_num_threads(nt):\n", - " \"Get numpy (and others) to use `nt` threads\"\n", - " try: import mkl; mkl.set_num_threads(nt)\n", - " except: pass\n", - " try: import torch; torch.set_num_threads(nt)\n", - " except: pass\n", - " os.environ['IPC_ENABLE']='1'\n", - " for o in ['OPENBLAS_NUM_THREADS','NUMEXPR_NUM_THREADS','OMP_NUM_THREADS','MKL_NUM_THREADS']:\n", - " os.environ[o] = str(nt)" + "_all_ = ['Self']" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "This sets the number of threads consistently for many tools, by:\n", + "This is a concise way to create lambdas that are calling methods on an object (note the capitalization!)\n", "\n", - "1. Set the following environment variables equal to `nt`: `OPENBLAS_NUM_THREADS`,`NUMEXPR_NUM_THREADS`,`OMP_NUM_THREADS`,`MKL_NUM_THREADS`\n", - "2. Sets `nt` threads for numpy and pytorch." + "`Self.sum()`, for instance, is a shortcut for `lambda o: o.sum()`." ] }, { @@ -4002,41 +2519,39 @@ "metadata": {}, "outputs": [], "source": [ - "#export\n", - "def _call(lock, pause, n, g, item):\n", - " l = False\n", - " if pause:\n", - " try:\n", - " l = lock.acquire(timeout=pause*(n+2))\n", - " time.sleep(pause)\n", - " finally:\n", - " if l: lock.release()\n", - " return g(item)" + "f = Self.sum()\n", + "x = np.array([3.,1])\n", + "test_eq(f(x), 4.)\n", + "\n", + "# This is equivalent to above\n", + "f = lambda o: o.sum()\n", + "x = np.array([3.,1])\n", + "test_eq(f(x), 4.)\n", + "\n", + "f = Self.argmin()\n", + "arr = np.array([1,2,3,4,5])\n", + "test_eq(f(arr), arr.argmin())\n", + "\n", + "f = Self.sum().is_integer()\n", + "x = np.array([3.,1])\n", + "test_eq(f(x), True)\n", + "\n", + "f = Self.sum().real.is_integer()\n", + "x = np.array([3.,1])\n", + "test_eq(f(x), True)\n", + "\n", + "f = Self.imag()\n", + "test_eq(f(3), 0)\n", + "\n", + "f = Self[1]\n", + "test_eq(f(x), 1)" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "#export\n", - "class ProcessPoolExecutor(concurrent.futures.ProcessPoolExecutor):\n", - " \"Same as Python's ProcessPoolExecutor, except can pass `max_workers==0` for serial execution\"\n", - " def __init__(self, max_workers=defaults.cpus, on_exc=print, pause=0, **kwargs):\n", - " if max_workers is None: max_workers=defaults.cpus\n", - " store_attr()\n", - " self.not_parallel = max_workers==0\n", - " if self.not_parallel: max_workers=1\n", - " super().__init__(max_workers, **kwargs)\n", - "\n", - " def map(self, f, items, timeout=None, chunksize=1, *args, **kwargs):\n", - " self.lock = Manager().Lock()\n", - " g = partial(f, *args, **kwargs)\n", - " if self.not_parallel: return map(g, items)\n", - " _g = partial(_call, self.lock, self.pause, self.max_workers, g)\n", - " try: return super().map(_g, items, timeout=timeout, chunksize=chunksize)\n", - " except Exception as e: self.on_exc(e)" + "## Other Helpers" ] }, { @@ -4046,22 +2561,9 @@ "outputs": [], "source": [ "#export\n", - "class ThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):\n", - " \"Same as Python's ThreadPoolExecutor, except can pass `max_workers==0` for serial execution\"\n", - " def __init__(self, max_workers=defaults.cpus, on_exc=print, pause=0, **kwargs):\n", - " if max_workers is None: max_workers=defaults.cpus\n", - " store_attr()\n", - " self.not_parallel = max_workers==0\n", - " if self.not_parallel: max_workers=1\n", - " super().__init__(max_workers, **kwargs)\n", - "\n", - " def map(self, f, items, timeout=None, chunksize=1, *args, **kwargs):\n", - " self.lock = Manager().Lock()\n", - " g = partial(f, *args, **kwargs)\n", - " if self.not_parallel: return map(g, items)\n", - " _g = partial(_call, self.lock, self.pause, self.max_workers, g)\n", - " try: return super().map(_g, items, timeout=timeout, chunksize=chunksize)\n", - " except Exception as e: self.on_exc(e)" + "class PrettyString(str):\n", + " \"Little hack to get strings to show properly in Jupyter.\"\n", + " def __repr__(self): return self" ] }, { @@ -4072,11 +2574,11 @@ { "data": { "text/markdown": [ - "

class ProcessPoolExecutor[source]

\n", + "

class PrettyString[source]

\n", "\n", - "> ProcessPoolExecutor(**`max_workers`**=*`4`*, **`on_exc`**=*`print`*, **`pause`**=*`0`*, **\\*\\*`kwargs`**) :: [`ProcessPoolExecutor`](/utils.html#ProcessPoolExecutor)\n", + "> PrettyString() :: `str`\n", "\n", - "Same as Python's ProcessPoolExecutor, except can pass `max_workers==0` for serial execution" + "Little hack to get strings to show properly in Jupyter." ], "text/plain": [ "" @@ -4087,38 +2589,42 @@ } ], "source": [ - "show_doc(ProcessPoolExecutor, title_level=4)" + "show_doc(PrettyString, title_level=4)" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "#export\n", - "try: from fastprogress import progress_bar\n", - "except: progress_bar = None" + "Allow strings with special characters to render properly in Jupyter. Without calling `print()` strings with special characters are displayed like so:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'a string\\nwith\\nnew\\nlines and\\ttabs'" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "#export \n", - "def parallel(f, items, *args, n_workers=defaults.cpus, total=None, progress=None, pause=0,\n", - " threadpool=False, timeout=None, chunksize=1, **kwargs):\n", - " \"Applies `func` in parallel to `items`, using `n_workers`\"\n", - " if progress is None: progress = progress_bar is not None\n", - " pool = ThreadPoolExecutor if threadpool else ProcessPoolExecutor\n", - " with pool(n_workers, pause=pause) as ex:\n", - " r = ex.map(f,items, *args, timeout=timeout, chunksize=chunksize, **kwargs)\n", - " if progress:\n", - " if total is None: total = len(items)\n", - " r = progress_bar(r, total=total, leave=False)\n", - " return L(r)" + "with_special_chars='a string\\nwith\\nnew\\nlines and\\ttabs'\n", + "with_special_chars" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can correct this with `PrettyString`:" ] }, { @@ -4128,53 +2634,46 @@ "outputs": [ { "data": { - "text/html": [], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [], "text/plain": [ - "" + "a string\n", + "with\n", + "new\n", + "lines and\ttabs" ] }, + "execution_count": null, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ - "def add_one(x, a=1): \n", - " time.sleep(random.random()/80)\n", - " return x+a\n", - "\n", - "inp,exp = range(50),range(1,51)\n", - "test_eq(parallel(add_one, inp, n_workers=2, progress=False), exp)\n", - "test_eq(parallel(add_one, inp, threadpool=True, n_workers=2, progress=False), exp)\n", - "test_eq(parallel(add_one, inp, n_workers=0), exp)\n", - "test_eq(parallel(add_one, inp, n_workers=1, a=2), range(2,52))\n", - "test_eq(parallel(add_one, inp, n_workers=0, a=2), range(2,52))" + "PrettyString(with_special_chars)" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def even_mults(start, stop, n):\n", + " \"Build log-stepped array from `start` to `stop` in `n` steps.\"\n", + " if n==1: return stop\n", + " mult = stop/start\n", + " step = mult**(1/(n-1))\n", + " return [start*(step**i) for i in range(n)]" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "Use the `pause` parameter to ensure a pause of `pause` seconds between processes starting. This is in case there are race conditions in starting some process, or to stagger the time each process starts, for example when making many requests to a webserver. Set `threadpool=True` to use `ThreadPoolExecutor` instead of `ProcessPoolExecutor`." + "test_eq(even_mults(2,8,3), [2,4,8])\n", + "test_eq(even_mults(2,32,5), [2,4,8,16,32])\n", + "test_eq(even_mults(2,8,1), 8)" ] }, { @@ -4183,7 +2682,13 @@ "metadata": {}, "outputs": [], "source": [ - "from datetime import datetime" + "#export\n", + "def num_cpus():\n", + " \"Get number of cpus\"\n", + " try: return len(os.sched_getaffinity(0))\n", + " except AttributeError: return os.cpu_count()\n", + "\n", + "defaults.cpus = num_cpus()" ] }, { @@ -4193,39 +2698,30 @@ "outputs": [ { "data": { - "text/html": [], "text/plain": [ - "" + "64" ] }, + "execution_count": null, "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1 2020-10-12 21:48:54.012317\n", - "0 2020-10-12 21:48:54.265181\n", - "2 2020-10-12 21:48:54.519284\n", - "3 2020-10-12 21:48:54.773004\n", - "4 2020-10-12 21:48:55.026344\n" - ] + "output_type": "execute_result" } ], "source": [ - "def print_time(i): \n", - " time.sleep(random.random()/1000)\n", - " print(i, datetime.now())\n", - "\n", - "parallel(print_time, range(5), n_workers=2, pause=0.25);" + "num_cpus()" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "Note that `f` should accept a collection of items." + "#export\n", + "def add_props(f, g=None, n=2):\n", + " \"Create properties passing each of `range(n)` to f\"\n", + " if g is None: return (property(partial(f,i)) for i in range(n))\n", + " return (property(partial(f,i), partial(g,i)) for i in range(n))" ] }, { @@ -4234,13 +2730,11 @@ "metadata": {}, "outputs": [], "source": [ - "#export\n", - "def run_procs(f, f_done, args):\n", - " \"Call `f` for each item in `args` in parallel, yielding `f_done`\"\n", - " processes = L(args).map(Process, args=arg0, target=f)\n", - " for o in processes: o.start()\n", - " yield from f_done()\n", - " processes.map(Self.join())" + "class _T(): a,b = add_props(lambda i,x:i*2)\n", + "\n", + "t = _T()\n", + "test_eq(t.a,0)\n", + "test_eq(t.b,2)" ] }, { @@ -4249,11 +2743,18 @@ "metadata": {}, "outputs": [], "source": [ - "#export\n", - "def _f_pg(obj, queue, batch, start_idx):\n", - " for i,b in enumerate(obj(batch)): queue.put((start_idx+i,b))\n", + "class _T(): \n", + " def __init__(self, v): self.v=v\n", + " def _set(i, self, v): self.v[i] = v\n", + " a,b = add_props(lambda i,x: x.v[i], _set)\n", "\n", - "def _done_pg(queue, items): return (queue.get() for _ in items)" + "t = _T([0,2])\n", + "test_eq(t.a,0)\n", + "test_eq(t.b,2)\n", + "t.a = t.a+1\n", + "t.b = 3\n", + "test_eq(t.a,1)\n", + "test_eq(t.b,3)" ] }, { @@ -4262,87 +2763,73 @@ "metadata": {}, "outputs": [], "source": [ - "#export \n", - "def parallel_gen(cls, items, n_workers=defaults.cpus, **kwargs):\n", - " \"Instantiate `cls` in `n_workers` procs & call each on a subset of `items` in parallel.\"\n", - " if n_workers==0:\n", - " yield from enumerate(list(cls(**kwargs)(items)))\n", - " return\n", - " batches = L(chunked(items, n_chunks=n_workers))\n", - " idx = L(itertools.accumulate(0 + batches.map(len)))\n", - " queue = Queue()\n", - " if progress_bar: items = progress_bar(items, leave=False)\n", - " f=partial(_f_pg, cls(**kwargs), queue)\n", - " done=partial(_done_pg, queue, items)\n", - " yield from run_procs(f, done, L(batches,idx).zip())" + "#export\n", + "def _typeerr(arg, val, typ): return TypeError(f\"{arg}=={val} not {typ}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "class _C:\n", - " def __call__(self, o): return ((i+1) for i in o)\n", - "\n", - "items = range(5)\n", + "#export\n", + "def typed(f):\n", + " \"Decorator to check param and return types at runtime\"\n", + " names = f.__code__.co_varnames\n", + " anno = f.__annotations__\n", + " ret = anno.pop('return',None)\n", + " def _f(*args,**kwargs):\n", + " kw = {**kwargs}\n", + " if len(anno) > 0:\n", + " for i,arg in enumerate(args): kw[names[i]] = arg\n", + " for k,v in kw.items():\n", + " if not isinstance(v,anno[k]): raise _typeerr(k, v, anno[k])\n", + " res = f(*args,**kwargs)\n", + " if ret is not None and not isinstance(res,ret): raise _typeerr(\"return\", res, ret)\n", + " return res\n", + " return functools.update_wrapper(_f, f)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`typed` validates argument types at **runtime**. This is in contrast to [MyPy](http://mypy-lang.org/) which only offers static type checking.\n", "\n", - "res = L(parallel_gen(_C, items, n_workers=3))\n", - "idxs,dat1 = zip(*res.sorted(itemgetter(0)))\n", - "test_eq(dat1, range(1,6))\n", + "For example, a `TypeError` will be raised if we try to pass an integer into the first argument of the below function: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@typed\n", + "def discount(price:int, pct:float): \n", + " return (1-pct) * price\n", "\n", - "res = L(parallel_gen(_C, items, n_workers=0))\n", - "idxs,dat2 = zip(*res.sorted(itemgetter(0)))\n", - "test_eq(dat2, dat1)" + "with ExceptionExpected(TypeError): discount(100.0, .1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "`cls` is any class with `__call__`. It will be passed `args` and `kwargs` when initialized. Note that `n_workers` instances of `cls` are created, one in each process. `items` are then split in `n_workers` batches and one is sent to each `cls`. The function then returns a generator of tuples of item indices and results." + "We can also optionally allow multiple types by enumarating the types in a tuple as illustrated below:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "class TestSleepyBatchFunc:\n", - " \"For testing parallel processes that run at different speeds\"\n", - " def __init__(self): self.a=1\n", - " def __call__(self, batch):\n", - " for k in batch:\n", - " time.sleep(random.random()/4)\n", - " yield k+self.a\n", + "def discount(price:(int,float), pct:float): \n", + " return (1-pct) * price\n", "\n", - "x = np.linspace(0,0.99,20)\n", - "res = L(parallel_gen(TestSleepyBatchFunc, x, n_workers=2))\n", - "test_eq(res.sorted().itemgot(1), x+1)" + "assert 90.0 == discount(100.0, .1)" ] }, { @@ -4351,15 +2838,19 @@ "metadata": {}, "outputs": [], "source": [ - "#export\n", - "def threaded(f):\n", - " \"Run `f` in a thread, and returns the thread\"\n", - " @wraps(f)\n", - " def _f(*args, **kwargs):\n", - " res = Thread(target=f, args=args, kwargs=kwargs)\n", - " res.start()\n", - " return res\n", - " return _f" + "@typed\n", + "def foo(a:int, b:str='a'): return a\n", + "test_eq(foo(1, '2'), 1)\n", + "\n", + "with ExceptionExpected(TypeError): foo(1,2)\n", + "\n", + "@typed\n", + "def foo()->str: return 1\n", + "with ExceptionExpected(TypeError): foo()\n", + "\n", + "@typed\n", + "def foo()->str: return '1'\n", + "assert foo()" ] }, { @@ -4503,7 +2994,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "These variables are availabe as booleans in `fastcore.utils` as `IN_IPYTHON`, `IN_JUPYTER`, `IN_COLAB` and `IN_NOTEBOOK`." + "These variables are availabe as booleans in `fastcore.basics` as `IN_IPYTHON`, `IN_JUPYTER`, `IN_COLAB` and `IN_NOTEBOOK`." ] }, { @@ -4543,9 +3034,10 @@ "output_type": "stream", "text": [ "Converted 00_test.ipynb.\n", - "Converted 01_foundation.ipynb.\n", - "Converted 02_utils.ipynb.\n", + "Converted 01_basics.ipynb.\n", + "Converted 02_foundation.ipynb.\n", "Converted 03_dispatch.ipynb.\n", + "Converted 03_xtras.ipynb.\n", "Converted 04_transform.ipynb.\n", "Converted 05_logargs.ipynb.\n", "Converted 06_meta.ipynb.\n", diff --git a/nbs/01_foundation.ipynb b/nbs/02_foundation.ipynb similarity index 96% rename from nbs/01_foundation.ipynb rename to nbs/02_foundation.ipynb index c0fb8740..0efe0eb2 100644 --- a/nbs/01_foundation.ipynb +++ b/nbs/02_foundation.ipynb @@ -17,6 +17,7 @@ "source": [ "#export\n", "from fastcore.imports import *\n", + "from fastcore.basics import *\n", "from functools import lru_cache\n", "from contextlib import contextmanager\n", "from copy import copy\n", @@ -45,16 +46,6 @@ "> Basic functions used in the fastai library" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "defaults = SimpleNamespace()" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -747,23 +738,6 @@ "test_eq(mask2idxs(array([1,2,3])), [1,2,3])" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def _is_array(x): return hasattr(x,'__array__') or hasattr(x,'iloc')\n", - "\n", - "def _listify(o):\n", - " if o is None: return []\n", - " if isinstance(o, list): return o\n", - " if isinstance(o, str) or _is_array(o): return [o]\n", - " if is_iter(o): return list(o)\n", - " return [o]" - ] - }, { "cell_type": "code", "execution_count": null, @@ -773,7 +747,7 @@ "#export\n", "def cycle(o):\n", " \"Like `itertools.cycle` except creates list of `None`s if `o` is empty\"\n", - " o = _listify(o)\n", + " o = listify(o)\n", " return itertools.cycle(o) if o is not None and len(o) > 0 else itertools.cycle([None])" ] }, @@ -1476,28 +1450,6 @@ "test_eq(range_of(4), [0,1,2,3])" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def sorted_ex(iterable, key=None, reverse=False):\n", - " \"Like `sorted`, but if key is str use `attrgetter`; if int use `itemgetter`\"\n", - " if isinstance(key,str): k=lambda o:getattr(o,key,0)\n", - " elif isinstance(key,int): k=itemgetter(key)\n", - " else: k=key\n", - " return sorted(iterable, key=k, reverse=reverse)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "See `L.sorted` for examples." - ] - }, { "cell_type": "code", "execution_count": null, @@ -1578,40 +1530,6 @@ "test_eq(nested_attr(a, 'b.d'), None)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def stop(e=StopIteration):\n", - " \"Raises exception `e` (by default `StopException`)\"\n", - " raise e\n", - "\n", - "def tst():\n", - " try: \n", - " stop() \n", - " except StopIteration: \n", - " return True\n", - " \n", - "def tst2():\n", - " try: \n", - " stop(e=ValueError) \n", - " except ValueError: \n", - " return True" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert tst()\n", - "assert tst2()" - ] - }, { "cell_type": "code", "execution_count": null, @@ -1686,8 +1604,8 @@ " def __init__(self, items=None, *rest, use_list=False, match=None):\n", " if rest: items = (items,)+rest\n", " if items is None: items = []\n", - " if (use_list is not None) or not _is_array(items):\n", - " items = list(items) if use_list else _listify(items)\n", + " if (use_list is not None) or not is_array(items):\n", + " items = list(items) if use_list else listify(items)\n", " if match is not None:\n", " if is_coll(match): match = len(match)\n", " if len(items)==1: items = items*match\n", @@ -1711,7 +1629,7 @@ " \"Set `idx` (can be list of indices, or mask, or int) items to `o` (which is broadcast if not iterable)\"\n", " if isinstance(idx, int): self.items[idx] = o\n", " else:\n", - " idx = idx if isinstance(idx,L) else _listify(idx)\n", + " idx = idx if isinstance(idx,L) else listify(idx)\n", " if not is_iter(o): o = [o]*len(idx)\n", " for i,o_ in zip(idx,o): self.items[i] = o_\n", "\n", @@ -1727,9 +1645,9 @@ " def __invert__(self): return self._new(not i for i in self)\n", " def __repr__(self): return repr(self.items)\n", " def _repr_pretty_(self, p, cycle):\n", - " p.text('...' if cycle else repr(self.items) if _is_array(self.items) else coll_repr(self))\n", + " p.text('...' if cycle else repr(self.items) if is_array(self.items) else coll_repr(self))\n", " def __mul__ (a,b): return a._new(a.items*b)\n", - " def __add__ (a,b): return a._new(a.items+_listify(b))\n", + " def __add__ (a,b): return a._new(a.items+listify(b))\n", " def __radd__(a,b): return a._new(b)+a\n", " def __addi__(a,b):\n", " a.items += list(b)\n", @@ -1831,7 +1749,7 @@ { "data": { "text/plain": [ - "b'\\x80\\x04\\x95\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x07inspect\\x94\\x8c\\tSignature\\x94\\x93\\x94(h\\x00\\x8c\\tParameter\\x94\\x93\\x94\\x8c\\x05items\\x94h\\x00\\x8c\\x0e_ParameterKind\\x94\\x93\\x94K\\x01\\x85\\x94R\\x94\\x86\\x94R\\x94}\\x94(\\x8c\\x08_default\\x94N\\x8c\\x0b_annotation\\x94h\\x00\\x8c\\x06_empty\\x94\\x93\\x94ubh\\x04\\x8c\\x04rest\\x94h\\x07K\\x02\\x85\\x94R\\x94\\x86\\x94R\\x94}\\x94(h\\rh\\x10h\\x0eh\\x10ubh\\x04\\x8c\\x08use_list\\x94h\\x07K\\x03\\x85\\x94R\\x94\\x86\\x94R\\x94}\\x94(h\\r\\x89h\\x0eh\\x10ubh\\x04\\x8c\\x05match\\x94h\\x19\\x86\\x94R\\x94}\\x94(h\\rNh\\x0eh\\x10ubt\\x94\\x85\\x94R\\x94}\\x94\\x8c\\x12_return_annotation\\x94h\\x10sb.'" + "b'\\x80\\x03cinspect\\nSignature\\nq\\x00(cinspect\\nParameter\\nq\\x01X\\x05\\x00\\x00\\x00itemsq\\x02cinspect\\n_ParameterKind\\nq\\x03K\\x01\\x85q\\x04Rq\\x05\\x86q\\x06Rq\\x07}q\\x08(X\\x08\\x00\\x00\\x00_defaultq\\tNX\\x0b\\x00\\x00\\x00_annotationq\\ncinspect\\n_empty\\nq\\x0bubh\\x01X\\x04\\x00\\x00\\x00restq\\x0ch\\x03K\\x02\\x85q\\rRq\\x0e\\x86q\\x0fRq\\x10}q\\x11(h\\th\\x0bh\\nh\\x0bubh\\x01X\\x08\\x00\\x00\\x00use_listq\\x12h\\x03K\\x03\\x85q\\x13Rq\\x14\\x86q\\x15Rq\\x16}q\\x17(h\\t\\x89h\\nh\\x0bubh\\x01X\\x05\\x00\\x00\\x00matchq\\x18h\\x14\\x86q\\x19Rq\\x1a}q\\x1b(h\\tNh\\nh\\x0bubtq\\x1c\\x85q\\x1dRq\\x1e}q\\x1fX\\x12\\x00\\x00\\x00_return_annotationq h\\x0bsb.'" ] }, "execution_count": null, @@ -1978,7 +1896,7 @@ { "data": { "text/plain": [ - "[10, 'j', 4]" + "[2, 'j', 1]" ] }, "execution_count": null, @@ -3176,9 +3094,10 @@ "output_type": "stream", "text": [ "Converted 00_test.ipynb.\n", - "Converted 01_foundation.ipynb.\n", - "Converted 02_utils.ipynb.\n", + "Converted 01_basics.ipynb.\n", + "Converted 02_foundation.ipynb.\n", "Converted 03_dispatch.ipynb.\n", + "Converted 03_xtras.ipynb.\n", "Converted 04_transform.ipynb.\n", "Converted 05_logargs.ipynb.\n", "Converted 06_meta.ipynb.\n", diff --git a/nbs/03_xtras.ipynb b/nbs/03_xtras.ipynb new file mode 100644 index 00000000..1464a487 --- /dev/null +++ b/nbs/03_xtras.ipynb @@ -0,0 +1,1708 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#default_exp xtras" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "from fastcore.imports import *\n", + "from fastcore.foundation import *\n", + "from fastcore.basics import *\n", + "from functools import wraps\n", + "\n", + "import mimetypes,bz2,pickle,random,json,urllib,subprocess,shlex,bz2,gzip,distutils.util\n", + "from contextlib import contextmanager,ExitStack\n", + "from pdb import set_trace\n", + "from urllib.request import Request,urlopen\n", + "from urllib.error import HTTPError\n", + "from urllib.parse import urlencode\n", + "from threading import Thread" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from fastcore.test import *\n", + "from nbdev.showdoc import *\n", + "from fastcore.nb_imports import *" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Utility functions\n", + "\n", + "> Utility functions used in the fastai library" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Collections" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def dict2obj(d):\n", + " \"Convert (possibly nested) dicts (or lists of dicts) to `AttrDict`\"\n", + " if isinstance(d, (L,list)): return L(d).map(dict2obj)\n", + " if not isinstance(d, dict): return d\n", + " return AttrDict(**{k:dict2obj(v) for k,v in d.items()})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This is a convenience to give you \"dotted\" access to (possibly nested) dictionaries, e.g:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'a': 1, 'b': {'c': 2, 'd': 3}}" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "d1 = dict(a=1, b=dict(c=2,d=3))\n", + "d2 = dict2obj(d1)\n", + "test_eq(d2.b.c, 2)\n", + "test_eq(d2.b['c'], 2)\n", + "d2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It can also be used on lists of dicts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds = L(d1, d1)\n", + "test_eq(dict2obj(ds)[0].b.c, 2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def tuplify(o, use_list=False, match=None):\n", + " \"Make `o` a tuple\"\n", + " return tuple(L(o, use_list=use_list, match=match))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "test_eq(tuplify(None),())\n", + "test_eq(tuplify([1,2,3]),(1,2,3))\n", + "test_eq(tuplify(1,match=[1,2,3]),(1,1,1))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def uniqueify(x, sort=False, bidir=False, start=None):\n", + " \"Unique elements in `x`, optionally `sort`-ed, optionally return reverse correspondence, optionally prepend with elements.\"\n", + " res = L(x).unique()\n", + " if start is not None: res = start+res\n", + " if sort: res.sort()\n", + " if bidir: return res, res.val2idx()\n", + " return res" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# test\n", + "test_eq(set(uniqueify([1,1,0,5,0,3])),{0,1,3,5})\n", + "test_eq(uniqueify([1,1,0,5,0,3], sort=True),[0,1,3,5])\n", + "test_eq(uniqueify([1,1,0,5,0,3], start=[7,8,6]), [7,8,6,1,0,5,3])\n", + "v,o = uniqueify([1,1,0,5,0,3], bidir=True)\n", + "test_eq(v,[1,0,5,3])\n", + "test_eq(o,{1:0, 0: 1, 5: 2, 3: 3})\n", + "v,o = uniqueify([1,1,0,5,0,3], sort=True, bidir=True)\n", + "test_eq(v,[0,1,3,5])\n", + "test_eq(o,{0:0, 1: 1, 3: 2, 5: 3})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def is_listy(x):\n", + " \"`isinstance(x, (tuple,list,L,slice,Generator))`\"\n", + " return isinstance(x, (tuple,list,L,slice,Generator))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert is_listy((1,))\n", + "assert is_listy([1])\n", + "assert is_listy(L([1]))\n", + "assert is_listy(slice(2))\n", + "assert not is_listy(array([1]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def shufflish(x, pct=0.04):\n", + " \"Randomly relocate items of `x` up to `pct` of `len(x)` from their starting location\"\n", + " n = len(x)\n", + " return L(x[i] for i in sorted(range_of(x), key=lambda o: o+n*(1+random.random()*pct)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def mapped(f, it):\n", + " \"map `f` over `it`, unless it's not listy, in which case return `f(it)`\"\n", + " return L(it).map(f) if is_listy(it) else f(it)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def _f(x,a=1): return x-a\n", + "\n", + "test_eq(mapped(_f,1),0)\n", + "test_eq(mapped(_f,[1,2]),[0,1])\n", + "test_eq(mapped(_f,(1,)),(0,))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Reindexing Collections" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "#hide\n", + "class IterLen:\n", + " \"Base class to add iteration to anything supporting `__len__` and `__getitem__`\"\n", + " def __iter__(self): return (self[i] for i in range_of(self))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "@docs\n", + "class ReindexCollection(GetAttr, IterLen):\n", + " \"Reindexes collection `coll` with indices `idxs` and optional LRU cache of size `cache`\"\n", + " _default='coll'\n", + " def __init__(self, coll, idxs=None, cache=None, tfm=noop):\n", + " if idxs is None: idxs = L.range(coll) \n", + " store_attr()\n", + " if cache is not None: self._get = functools.lru_cache(maxsize=cache)(self._get)\n", + "\n", + " def _get(self, i): return self.tfm(self.coll[i])\n", + " def __getitem__(self, i): return self._get(self.idxs[i])\n", + " def __len__(self): return len(self.coll)\n", + " def reindex(self, idxs): self.idxs = idxs\n", + " def shuffle(self): random.shuffle(self.idxs)\n", + " def cache_clear(self): self._get.cache_clear()\n", + " def __getstate__(self): return {'coll': self.coll, 'idxs': self.idxs, 'cache': self.cache, 'tfm': self.tfm}\n", + " def __setstate__(self, s): self.coll,self.idxs,self.cache,self.tfm = s['coll'],s['idxs'],s['cache'],s['tfm']\n", + "\n", + " _docs = dict(reindex=\"Replace `self.idxs` with idxs\",\n", + " shuffle=\"Randomly shuffle indices\",\n", + " cache_clear=\"Clear LRU cache\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "

class ReindexCollection[source]

\n", + "\n", + "> ReindexCollection(**`coll`**, **`idxs`**=*`None`*, **`cache`**=*`None`*, **`tfm`**=*`noop`*) :: [`GetAttr`](/foundation.html#GetAttr)\n", + "\n", + "Reindexes collection `coll` with indices `idxs` and optional LRU cache of size `cache`" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "show_doc(ReindexCollection, title_level=4)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This is useful when constructing batches or organizing data in a particular manner (i.e. for deep learning). This class is primarly used in organizing data for language models in fastai." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can supply a custom index upon instantiation with the `idxs` argument, or you can call the `reindex` method to supply a new index for your collection.\n", + "\n", + "Here is how you can reindex a list such that the elements are reversed:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['e', 'd', 'c', 'b', 'a']" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rc=ReindexCollection(['a', 'b', 'c', 'd', 'e'], idxs=[4,3,2,1,0])\n", + "list(rc)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Alternatively, you can use the `reindex` method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "
ReindexCollection.reindex[source]
\n", + "\n", + "> ReindexCollection.reindex(**`idxs`**)\n", + "\n", + "Replace `self.idxs` with idxs" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "show_doc(ReindexCollection.reindex, title_level=6)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['e', 'd', 'c', 'b', 'a']" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rc=ReindexCollection(['a', 'b', 'c', 'd', 'e'])\n", + "rc.reindex([4,3,2,1,0])\n", + "list(rc)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can optionally specify a LRU cache, which uses [functools.lru_cache](https://docs.python.org/3/library/functools.html#functools.lru_cache) upon instantiation:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "CacheInfo(hits=1, misses=1, maxsize=2, currsize=1)" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sz = 50\n", + "t = ReindexCollection(L.range(sz), cache=2)\n", + "\n", + "#trigger a cache hit by indexing into the same element multiple times\n", + "t[0], t[0]\n", + "t._get.cache_info()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can optionally clear the LRU cache by calling the `cache_clear` method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "
ReindexCollection.cache_clear[source]
\n", + "\n", + "> ReindexCollection.cache_clear()\n", + "\n", + "Clear LRU cache" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "show_doc(ReindexCollection.cache_clear, title_level=5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "CacheInfo(hits=0, misses=0, maxsize=2, currsize=0)" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sz = 50\n", + "t = ReindexCollection(L.range(sz), cache=2)\n", + "\n", + "#trigger a cache hit by indexing into the same element multiple times\n", + "t[0], t[0]\n", + "t.cache_clear()\n", + "t._get.cache_info()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "
ReindexCollection.shuffle[source]
\n", + "\n", + "> ReindexCollection.shuffle()\n", + "\n", + "Randomly shuffle indices" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "show_doc(ReindexCollection.shuffle, title_level=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that an ordered index is automatically constructed for the data structure even if one is not supplied." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['e', 'h', 'a', 'd', 'f', 'g', 'b', 'c']" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rc=ReindexCollection(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'])\n", + "rc.shuffle()\n", + "list(rc)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sz = 50\n", + "t = ReindexCollection(L.range(sz), cache=2)\n", + "test_eq(list(t), range(sz))\n", + "test_eq(t[sz-1], sz-1)\n", + "test_eq(t._get.cache_info().hits, 1)\n", + "t.shuffle()\n", + "test_eq(t._get.cache_info().hits, 1)\n", + "test_ne(list(t), range(sz))\n", + "test_eq(set(t), set(range(sz)))\n", + "t.cache_clear()\n", + "test_eq(t._get.cache_info().hits, 0)\n", + "test_eq(t.count(0), 1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#hide\n", + "#Test ReindexCollection pickles\n", + "t1 = pickle.loads(pickle.dumps(t))\n", + "test_eq(list(t), list(t1))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Extensions to Pathlib.Path" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "An extension of the standard python libary [Pathlib.Path](https://docs.python.org/3/library/pathlib.html#basic-use). These extensions are accomplished by monkey patching additional methods onto `Pathlib.Path`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "@patch\n", + "def readlines(self:Path, hint=-1, encoding='utf8'):\n", + " \"Read the content of `self`\"\n", + " with self.open(encoding=encoding) as f: return f.readlines(hint)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "@patch\n", + "def mk_write(self:Path, data, encoding=None, errors=None, mode=511):\n", + " \"Make all parent dirs of `self`\"\n", + " self.parent.mkdir(exist_ok=True, parents=True, mode=mode)\n", + " self.write_text(data, encoding=encoding, errors=errors)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "@patch\n", + "def ls(self:Path, n_max=None, file_type=None, file_exts=None):\n", + " \"Contents of path as a list\"\n", + " extns=L(file_exts)\n", + " if file_type: extns += L(k for k,v in mimetypes.types_map.items() if v.startswith(file_type+'/'))\n", + " has_extns = len(extns)==0\n", + " res = (o for o in self.iterdir() if has_extns or o.suffix in extns)\n", + " if n_max is not None: res = itertools.islice(res, n_max)\n", + " return L(res)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We add an `ls()` method to `pathlib.Path` which is simply defined as `list(Path.iterdir())`, mainly for convenience in REPL environments such as notebooks." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Path('.ipynb_checkpoints')" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "path = Path()\n", + "t = path.ls()\n", + "assert len(t)>0\n", + "t1 = path.ls(10)\n", + "test_eq(len(t1), 10)\n", + "t2 = path.ls(file_exts='.ipynb')\n", + "assert len(t)>len(t2)\n", + "t[0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also pass an optional `file_type` MIME prefix and/or a list of file extensions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(Path('../fastcore/logargs.py'), Path('04_transform.ipynb'))" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lib_path = (path/'../fastcore')\n", + "txt_files=lib_path.ls(file_type='text')\n", + "assert len(txt_files) > 0 and txt_files[0].suffix=='.py'\n", + "ipy_files=path.ls(file_exts=['.ipynb'])\n", + "assert len(ipy_files) > 0 and ipy_files[0].suffix=='.ipynb'\n", + "txt_files[0],ipy_files[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#hide\n", + "path = Path()\n", + "pkl = pickle.dumps(path)\n", + "p2 = pickle.loads(pkl)\n", + "test_eq(path.ls()[0], p2.ls()[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def open_file(fn, mode='r'):\n", + " \"Open a file, with optional compression if gz or bz2 suffix\"\n", + " if isinstance(fn, io.IOBase): return fn\n", + " fn = Path(fn)\n", + " if fn.suffix=='.bz2': return bz2.BZ2File(fn, mode)\n", + " elif fn.suffix=='.gz' : return gzip.GzipFile(fn, mode)\n", + " else: return open(fn,mode)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def save_pickle(fn, o):\n", + " \"Save a pickle file, to a file name or opened file\"\n", + " with open_file(fn, 'wb') as f: pickle.dump(o, f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def load_pickle(fn):\n", + " \"Load a pickle file from a file name or opened file\"\n", + " with open_file(fn, 'rb') as f: return pickle.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for suf in '.pkl','.bz2','.gz':\n", + " with tempfile.NamedTemporaryFile(suffix=suf) as f:\n", + " fn = Path(f.name)\n", + " save_pickle(fn, 't')\n", + " t = load_pickle(fn)\n", + " test_eq(t,'t')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "@patch\n", + "def __repr__(self:Path):\n", + " b = getattr(Path, 'BASE_PATH', None)\n", + " if b:\n", + " try: self = self.relative_to(b)\n", + " except: pass\n", + " return f\"Path({self.as_posix()!r})\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "fastai also updates the `repr` of `Path` such that, if `Path.BASE_PATH` is defined, all paths are printed relative to that path (as long as they are contained in `Path.BASE_PATH`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "t = ipy_files[0].absolute()\n", + "try:\n", + " Path.BASE_PATH = t.parent.parent\n", + " test_eq(repr(t), f\"Path('nbs/{t.name}')\")\n", + "finally: Path.BASE_PATH = None" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## File Functions\n", + "\n", + "Utilities (other than extensions to Pathlib.Path) for dealing with IO." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def bunzip(fn):\n", + " \"bunzip `fn`, raising exception if output already exists\"\n", + " fn = Path(fn)\n", + " assert fn.exists(), f\"{fn} doesn't exist\"\n", + " out_fn = fn.with_suffix('')\n", + " assert not out_fn.exists(), f\"{out_fn} already exists\"\n", + " with bz2.BZ2File(fn, 'rb') as src, out_fn.open('wb') as dst:\n", + " for d in iter(lambda: src.read(1024*1024), b''): dst.write(d)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "f = Path('files/test.txt')\n", + "if f.exists(): f.unlink()\n", + "bunzip('files/test.txt.bz2')\n", + "t = f.open().readlines()\n", + "test_eq(len(t),1)\n", + "test_eq(t[0], 'test\\n')\n", + "f.unlink()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def join_path_file(file, path, ext=''):\n", + " \"Return `path/file` if file is a string or a `Path`, file otherwise\"\n", + " if not isinstance(file, (str, Path)): return file\n", + " path.mkdir(parents=True, exist_ok=True)\n", + " return path/f'{file}{ext}'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "path = Path.cwd()/'_tmp'/'tst'\n", + "f = join_path_file('tst.txt', path)\n", + "assert path.exists()\n", + "test_eq(f, path/'tst.txt')\n", + "with open(f, 'w') as f_: assert join_path_file(f_, path) == f_\n", + "shutil.rmtree(Path.cwd()/'_tmp')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def urlread(url, data=None, **kwargs):\n", + " \"Retrieve `url`, using `data` dict or `kwargs` to `POST` if present\"\n", + " if kwargs and not data: data=kwargs\n", + " if data is not None:\n", + " if not isinstance(data, (str,bytes)): data = urlencode(data)\n", + " if not isinstance(data, bytes): data = data.encode('ascii')\n", + " cls = urllib.request.Request\n", + " if not isinstance(url,cls): url = cls(url)\n", + " url.headers['User-Agent'] = 'Mozilla/5.0'\n", + " with urlopen(url, data=data) as res: return res.read()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def urljson(url, data=None):\n", + " \"Retrieve `url` and decode json\"\n", + " return json.loads(urlread(url, data=data))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def run(cmd, *rest, ignore_ex=False, as_bytes=False):\n", + " \"Pass `cmd` (splitting with `shlex` if string) to `subprocess.run`; return `stdout`; raise `IOError` if fails\"\n", + " if rest: cmd = (cmd,)+rest\n", + " elif isinstance(cmd,str): cmd = shlex.split(cmd)\n", + " res = subprocess.run(cmd, capture_output=True)\n", + " stdout = res.stdout\n", + " if not as_bytes: stdout = stdout.decode()\n", + " if ignore_ex: return (res.returncode, stdout)\n", + " if res.returncode: raise IOError(\"{} ;; {}\".format(res.stdout, res.stderr))\n", + " return stdout" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can pass a string (which will be split based on standard shell rules), a list, or pass args directly:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert 'ipynb' in run('ls -l')\n", + "assert 'ipynb' in run(['ls', '-l'])\n", + "assert 'ipynb' in run('ls', '-l')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Some commands fail in non-error situations, like `grep`. Use `ignore_ex` in those cases, which will return a tuple of stdout and returncode:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "test_eq(run('grep asdfds 00_test.ipynb', ignore_ex=True)[0], 1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`run` automatically decodes returned bytes to a `str`. Use `as_bytes` to skip that:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "test_eq(run('echo hi', as_bytes=True), b'hi\\n')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def do_request(url, post=False, headers=None, **data):\n", + " \"Call GET or json-encoded POST on `url`, depending on `post`\"\n", + " if data:\n", + " if post: data = json.dumps(data).encode('ascii')\n", + " else:\n", + " url += \"?\" + urlencode(data)\n", + " data = None\n", + " return urljson(Request(url, headers=headers, data=data or None))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Other Helpers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def _is_instance(f, gs):\n", + " tst = [g if type(g) in [type, 'function'] else g.__class__ for g in gs]\n", + " for g in tst:\n", + " if isinstance(f, g) or f==g: return True\n", + " return False\n", + "\n", + "def _is_first(f, gs):\n", + " for o in L(getattr(f, 'run_after', None)):\n", + " if _is_instance(o, gs): return False\n", + " for g in gs:\n", + " if _is_instance(f, L(getattr(g, 'run_before', None))): return False\n", + " return True\n", + "\n", + "def sort_by_run(fs):\n", + " end = L(fs).attrgot('toward_end')\n", + " inp,res = L(fs)[~end] + L(fs)[end], L()\n", + " while len(inp):\n", + " for i,o in enumerate(inp):\n", + " if _is_first(o, inp):\n", + " res.append(inp.pop(i))\n", + " break\n", + " else: raise Exception(\"Impossible to sort\")\n", + " return res" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Transforms and callbacks will have run_after/run_before attributes, this function will sort them to respect those requirements (if it's possible). Also, sometimes we want a tranform/callback to be run at the end, but still be able to use run_after/run_before behaviors. For those, the function checks for a toward_end attribute (that needs to be True)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class Tst(): pass \n", + "class Tst1(): run_before=[Tst]\n", + "class Tst2():\n", + " run_before=Tst\n", + " run_after=Tst1\n", + " \n", + "tsts = [Tst(), Tst1(), Tst2()]\n", + "test_eq(sort_by_run(tsts), [tsts[1], tsts[2], tsts[0]])\n", + "\n", + "Tst2.run_before,Tst2.run_after = Tst1,Tst\n", + "test_fail(lambda: sort_by_run([Tst(), Tst1(), Tst2()]))\n", + "\n", + "def tst1(x): return x\n", + "tst1.run_before = Tst\n", + "test_eq(sort_by_run([tsts[0], tst1]), [tst1, tsts[0]])\n", + " \n", + "class Tst1():\n", + " toward_end=True\n", + "class Tst2():\n", + " toward_end=True\n", + " run_before=Tst1\n", + "tsts = [Tst(), Tst1(), Tst2()]\n", + "test_eq(sort_by_run(tsts), [tsts[0], tsts[2], tsts[1]])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def trace(f):\n", + " \"Add `set_trace` to an existing function `f`\"\n", + " if getattr(f, '_traced', False): return f\n", + " def _inner(*args,**kwargs):\n", + " set_trace()\n", + " return f(*args,**kwargs)\n", + " _inner._traced = True\n", + " return _inner" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can add a breakpoint to an existing function, e.g:\n", + "\n", + "```python\n", + "Path.cwd = trace(Path.cwd)\n", + "Path.cwd()\n", + "```\n", + "\n", + "Now, when the function is called it will drop you into the debugger. Note, you must issue the `s` command when you begin to step into the function that is being traced." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def round_multiple(x, mult, round_down=False):\n", + " \"Round `x` to nearest multiple of `mult`\"\n", + " def _f(x_): return (int if round_down else round)(x_/mult)*mult\n", + " res = L(x).map(_f)\n", + " return res if is_listy(x) else res[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "test_eq(round_multiple(63,32), 64)\n", + "test_eq(round_multiple(50,32), 64)\n", + "test_eq(round_multiple(40,32), 32)\n", + "test_eq(round_multiple( 0,32), 0)\n", + "test_eq(round_multiple(63,32, round_down=True), 32)\n", + "test_eq(round_multiple((63,40),32), (64,32))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "@contextmanager\n", + "def modified_env(*delete, **replace):\n", + " \"Context manager temporarily modifying `os.environ` by deleting `delete` and replacing `replace`\"\n", + " prev = dict(os.environ)\n", + " try:\n", + " os.environ.update(replace)\n", + " for k in delete: os.environ.pop(k, None)\n", + " yield\n", + " finally:\n", + " os.environ.clear()\n", + " os.environ.update(prev)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "oldsh = os.environ['SHELL']\n", + "\n", + "with modified_env('PATH', SHELL='a'):\n", + " test_eq(os.environ['SHELL'], 'a')\n", + " assert 'PATH' not in os.environ\n", + "\n", + "assert 'PATH' in os.environ\n", + "test_eq(os.environ['SHELL'], oldsh)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "class ContextManagers(GetAttr):\n", + " \"Wrapper for `contextlib.ExitStack` which enters a collection of context managers\"\n", + " def __init__(self, mgrs): self.default,self.stack = L(mgrs),ExitStack()\n", + " def __enter__(self): self.default.map(self.stack.enter_context)\n", + " def __exit__(self, *args, **kwargs): self.stack.__exit__(*args, **kwargs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "

class ContextManagers[source]

\n", + "\n", + "> ContextManagers(**`mgrs`**) :: [`GetAttr`](/foundation.html#GetAttr)\n", + "\n", + "Wrapper for `contextlib.ExitStack` which enters a collection of context managers" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "show_doc(ContextManagers, title_level=4)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def str2bool(s):\n", + " \"Case-insensitive convert string `s` too a bool (`y`,`yes`,`t`,`true`,`on`,`1`->`True`)\"\n", + " if not isinstance(s,str): return bool(s)\n", + " return bool(distutils.util.strtobool(s)) if s else False" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for o in \"y YES t True on 1\".split(): assert str2bool(o)\n", + "for o in \"n no FALSE off 0\".split(): assert not str2bool(o)\n", + "for o in 0,None,'',False: assert not str2bool(o)\n", + "for o in 1,True: assert str2bool(o)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multiprocessing" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "from multiprocessing import Process, Queue\n", + "import concurrent.futures\n", + "import time\n", + "from multiprocessing import Manager" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def set_num_threads(nt):\n", + " \"Get numpy (and others) to use `nt` threads\"\n", + " try: import mkl; mkl.set_num_threads(nt)\n", + " except: pass\n", + " try: import torch; torch.set_num_threads(nt)\n", + " except: pass\n", + " os.environ['IPC_ENABLE']='1'\n", + " for o in ['OPENBLAS_NUM_THREADS','NUMEXPR_NUM_THREADS','OMP_NUM_THREADS','MKL_NUM_THREADS']:\n", + " os.environ[o] = str(nt)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This sets the number of threads consistently for many tools, by:\n", + "\n", + "1. Set the following environment variables equal to `nt`: `OPENBLAS_NUM_THREADS`,`NUMEXPR_NUM_THREADS`,`OMP_NUM_THREADS`,`MKL_NUM_THREADS`\n", + "2. Sets `nt` threads for numpy and pytorch." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def _call(lock, pause, n, g, item):\n", + " l = False\n", + " if pause:\n", + " try:\n", + " l = lock.acquire(timeout=pause*(n+2))\n", + " time.sleep(pause)\n", + " finally:\n", + " if l: lock.release()\n", + " return g(item)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "class ProcessPoolExecutor(concurrent.futures.ProcessPoolExecutor):\n", + " \"Same as Python's ProcessPoolExecutor, except can pass `max_workers==0` for serial execution\"\n", + " def __init__(self, max_workers=defaults.cpus, on_exc=print, pause=0, **kwargs):\n", + " if max_workers is None: max_workers=defaults.cpus\n", + " store_attr()\n", + " self.not_parallel = max_workers==0\n", + " if self.not_parallel: max_workers=1\n", + " super().__init__(max_workers, **kwargs)\n", + "\n", + " def map(self, f, items, timeout=None, chunksize=1, *args, **kwargs):\n", + " self.lock = Manager().Lock()\n", + " g = partial(f, *args, **kwargs)\n", + " if self.not_parallel: return map(g, items)\n", + " _g = partial(_call, self.lock, self.pause, self.max_workers, g)\n", + " try: return super().map(_g, items, timeout=timeout, chunksize=chunksize)\n", + " except Exception as e: self.on_exc(e)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "class ThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):\n", + " \"Same as Python's ThreadPoolExecutor, except can pass `max_workers==0` for serial execution\"\n", + " def __init__(self, max_workers=defaults.cpus, on_exc=print, pause=0, **kwargs):\n", + " if max_workers is None: max_workers=defaults.cpus\n", + " store_attr()\n", + " self.not_parallel = max_workers==0\n", + " if self.not_parallel: max_workers=1\n", + " super().__init__(max_workers, **kwargs)\n", + "\n", + " def map(self, f, items, timeout=None, chunksize=1, *args, **kwargs):\n", + " self.lock = Manager().Lock()\n", + " g = partial(f, *args, **kwargs)\n", + " if self.not_parallel: return map(g, items)\n", + " _g = partial(_call, self.lock, self.pause, self.max_workers, g)\n", + " try: return super().map(_g, items, timeout=timeout, chunksize=chunksize)\n", + " except Exception as e: self.on_exc(e)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "

class ProcessPoolExecutor[source]

\n", + "\n", + "> ProcessPoolExecutor(**`max_workers`**=*`64`*, **`on_exc`**=*`print`*, **`pause`**=*`0`*, **\\*\\*`kwargs`**) :: [`ProcessPoolExecutor`](/xtras.html#ProcessPoolExecutor)\n", + "\n", + "Same as Python's ProcessPoolExecutor, except can pass `max_workers==0` for serial execution" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "show_doc(ProcessPoolExecutor, title_level=4)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "try: from fastprogress import progress_bar\n", + "except: progress_bar = None" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export \n", + "def parallel(f, items, *args, n_workers=defaults.cpus, total=None, progress=None, pause=0,\n", + " threadpool=False, timeout=None, chunksize=1, **kwargs):\n", + " \"Applies `func` in parallel to `items`, using `n_workers`\"\n", + " if progress is None: progress = progress_bar is not None\n", + " pool = ThreadPoolExecutor if threadpool else ProcessPoolExecutor\n", + " with pool(n_workers, pause=pause) as ex:\n", + " r = ex.map(f,items, *args, timeout=timeout, chunksize=chunksize, **kwargs)\n", + " if progress:\n", + " if total is None: total = len(items)\n", + " r = progress_bar(r, total=total, leave=False)\n", + " return L(r)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def add_one(x, a=1): \n", + " time.sleep(random.random()/80)\n", + " return x+a\n", + "\n", + "inp,exp = range(50),range(1,51)\n", + "test_eq(parallel(add_one, inp, n_workers=2, progress=False), exp)\n", + "test_eq(parallel(add_one, inp, threadpool=True, n_workers=2, progress=False), exp)\n", + "test_eq(parallel(add_one, inp, n_workers=0), exp)\n", + "test_eq(parallel(add_one, inp, n_workers=1, a=2), range(2,52))\n", + "test_eq(parallel(add_one, inp, n_workers=0, a=2), range(2,52))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use the `pause` parameter to ensure a pause of `pause` seconds between processes starting. This is in case there are race conditions in starting some process, or to stagger the time each process starts, for example when making many requests to a webserver. Set `threadpool=True` to use `ThreadPoolExecutor` instead of `ProcessPoolExecutor`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 2020-10-24 14:12:11.014495\n", + "1 2020-10-24 14:12:11.265213\n", + "2 2020-10-24 14:12:11.515864\n", + "3 2020-10-24 14:12:11.767123\n", + "4 2020-10-24 14:12:12.017351\n" + ] + } + ], + "source": [ + "def print_time(i): \n", + " time.sleep(random.random()/1000)\n", + " print(i, datetime.now())\n", + "\n", + "parallel(print_time, range(5), n_workers=2, pause=0.25);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that `f` should accept a collection of items." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def run_procs(f, f_done, args):\n", + " \"Call `f` for each item in `args` in parallel, yielding `f_done`\"\n", + " processes = L(args).map(Process, args=arg0, target=f)\n", + " for o in processes: o.start()\n", + " yield from f_done()\n", + " processes.map(Self.join())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def _f_pg(obj, queue, batch, start_idx):\n", + " for i,b in enumerate(obj(batch)): queue.put((start_idx+i,b))\n", + "\n", + "def _done_pg(queue, items): return (queue.get() for _ in items)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export \n", + "def parallel_gen(cls, items, n_workers=defaults.cpus, **kwargs):\n", + " \"Instantiate `cls` in `n_workers` procs & call each on a subset of `items` in parallel.\"\n", + " if n_workers==0:\n", + " yield from enumerate(list(cls(**kwargs)(items)))\n", + " return\n", + " batches = L(chunked(items, n_chunks=n_workers))\n", + " idx = L(itertools.accumulate(0 + batches.map(len)))\n", + " queue = Queue()\n", + " if progress_bar: items = progress_bar(items, leave=False)\n", + " f=partial(_f_pg, cls(**kwargs), queue)\n", + " done=partial(_done_pg, queue, items)\n", + " yield from run_procs(f, done, L(batches,idx).zip())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "class _C:\n", + " def __call__(self, o): return ((i+1) for i in o)\n", + "\n", + "items = range(5)\n", + "\n", + "res = L(parallel_gen(_C, items, n_workers=3))\n", + "idxs,dat1 = zip(*res.sorted(itemgetter(0)))\n", + "test_eq(dat1, range(1,6))\n", + "\n", + "res = L(parallel_gen(_C, items, n_workers=0))\n", + "idxs,dat2 = zip(*res.sorted(itemgetter(0)))\n", + "test_eq(dat2, dat1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`cls` is any class with `__call__`. It will be passed `args` and `kwargs` when initialized. Note that `n_workers` instances of `cls` are created, one in each process. `items` are then split in `n_workers` batches and one is sent to each `cls`. The function then returns a generator of tuples of item indices and results." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "class TestSleepyBatchFunc:\n", + " \"For testing parallel processes that run at different speeds\"\n", + " def __init__(self): self.a=1\n", + " def __call__(self, batch):\n", + " for k in batch:\n", + " time.sleep(random.random()/4)\n", + " yield k+self.a\n", + "\n", + "x = np.linspace(0,0.99,20)\n", + "res = L(parallel_gen(TestSleepyBatchFunc, x, n_workers=2))\n", + "test_eq(res.sorted().itemgot(1), x+1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#export\n", + "def threaded(f):\n", + " \"Run `f` in a thread, and returns the thread\"\n", + " @wraps(f)\n", + " def _f(*args, **kwargs):\n", + " res = Thread(target=f, args=args, kwargs=kwargs)\n", + " res.start()\n", + " return res\n", + " return _f" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Export -" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Converted 00_test.ipynb.\n", + "Converted 01_basics.ipynb.\n", + "Converted 02_foundation.ipynb.\n", + "Converted 03_dispatch.ipynb.\n", + "Converted 03_xtras.ipynb.\n", + "Converted 04_transform.ipynb.\n", + "Converted 05_logargs.ipynb.\n", + "Converted 06_meta.ipynb.\n", + "Converted 07_script.ipynb.\n", + "Converted index.ipynb.\n" + ] + } + ], + "source": [ + "#hide\n", + "from nbdev.export import notebook2script\n", + "notebook2script()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "jupytext": { + "split_at_heading": true + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/nbs/03_dispatch.ipynb b/nbs/04_dispatch.ipynb similarity index 100% rename from nbs/03_dispatch.ipynb rename to nbs/04_dispatch.ipynb diff --git a/nbs/04_transform.ipynb b/nbs/05_transform.ipynb similarity index 100% rename from nbs/04_transform.ipynb rename to nbs/05_transform.ipynb diff --git a/nbs/05_logargs.ipynb b/nbs/06_logargs.ipynb similarity index 100% rename from nbs/05_logargs.ipynb rename to nbs/06_logargs.ipynb diff --git a/nbs/06_meta.ipynb b/nbs/07_meta.ipynb similarity index 100% rename from nbs/06_meta.ipynb rename to nbs/07_meta.ipynb diff --git a/nbs/07_script.ipynb b/nbs/08_script.ipynb similarity index 100% rename from nbs/07_script.ipynb rename to nbs/08_script.ipynb