From aa04b3fdddf1eb8f50c8ceae50fb24f2497e3606 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Fri, 2 May 2014 18:16:02 -0400 Subject: [PATCH 001/146] Add `cytoolz.utils.dev_skip_test` decorator to skip tests for dev versions. These tests depend on `toolz`, and it is annoying when `toolz` changes in the middle of a PR. Now, the things these tests check for only need to be updated before being uploaded to PyPI. --- cytoolz/tests/test_curried_toolzlike.py | 4 ++++ cytoolz/tests/test_docstrings.py | 3 ++- cytoolz/tests/test_embedded_sigs.py | 3 +++ cytoolz/utils.py | 8 ++++++++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/cytoolz/tests/test_curried_toolzlike.py b/cytoolz/tests/test_curried_toolzlike.py index b45e3f7..3019bc2 100644 --- a/cytoolz/tests/test_curried_toolzlike.py +++ b/cytoolz/tests/test_curried_toolzlike.py @@ -3,16 +3,19 @@ import toolz import toolz.curried import types +from cytoolz.utils import dev_skip_test # Note that the tests in this file assume `toolz.curry` is a class, but we # may some day make `toolz.curry` a function and `toolz.Curry` a class. +@dev_skip_test def test_toolzcurry_is_class(): assert isinstance(toolz.curry, type) is True assert isinstance(toolz.curry, types.FunctionType) is False +@dev_skip_test def test_cytoolz_like_toolz(): for key, val in toolz.curried.__dict__.items(): if isinstance(val, toolz.curry): @@ -22,6 +25,7 @@ def test_cytoolz_like_toolz(): 'cytoolz.curried.%s should be curried' % key) +@dev_skip_test def test_toolz_like_cytoolz(): for key, val in cytoolz.curried.__dict__.items(): if isinstance(val, cytoolz.curry): diff --git a/cytoolz/tests/test_docstrings.py b/cytoolz/tests/test_docstrings.py index 18cdd93..11623c1 100644 --- a/cytoolz/tests/test_docstrings.py +++ b/cytoolz/tests/test_docstrings.py @@ -3,7 +3,7 @@ import toolz from cytoolz import curry, identity, keyfilter, valfilter, merge_with -from cytoolz.utils import raises +from cytoolz.utils import raises, dev_skip_test # `cytoolz` functions for which "# doctest: +SKIP" were added. @@ -30,6 +30,7 @@ def convertdoc(doc): return doc +@dev_skip_test def test_docstrings_uptodate(): differ = difflib.Differ() diff --git a/cytoolz/tests/test_embedded_sigs.py b/cytoolz/tests/test_embedded_sigs.py index 07fbb1b..a52e7e5 100644 --- a/cytoolz/tests/test_embedded_sigs.py +++ b/cytoolz/tests/test_embedded_sigs.py @@ -4,6 +4,7 @@ from types import BuiltinFunctionType from cytoolz import curry, identity, keyfilter, valfilter, merge_with +from cytoolz.utils import dev_skip_test @curry @@ -12,6 +13,7 @@ def isfrommod(modname, func): return modname in mod +@dev_skip_test def test_class_sigs(): """ Test that all ``cdef class`` extension types in ``cytoolz`` have correctly embedded the function signature as done in ``toolz``. @@ -54,6 +56,7 @@ def test_class_sigs(): aliases = {'comp': 'compose'} +@dev_skip_test def test_sig_at_beginning(): """ Test that the function signature is at the beginning of the docstring and is followed by exactly one blank line. diff --git a/cytoolz/utils.py b/cytoolz/utils.py index 2d1f6e7..1d04349 100644 --- a/cytoolz/utils.py +++ b/cytoolz/utils.py @@ -1,5 +1,6 @@ import doctest import inspect +import nose.tools import os.path import cytoolz @@ -104,3 +105,10 @@ def module_doctest(m, *args, **kwargs): """ fix_module_doctest(m) return doctest.testmod(m, *args, **kwargs).failed == 0 + + +# Decorator used to skip tests for developmental versions of CyToolz +if 'dev' in cytoolz.__version__: + dev_skip_test = nose.tools.nottest +else: + dev_skip_test = nose.tools.istest From 2022459b6530bbb7d0dd6aaa55349b0e3995a185 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 3 May 2014 16:58:32 -0400 Subject: [PATCH 002/146] Move `dev_skip_test` into a module in "tests/" directory. It was in `cytoolz.utils`, which added `nose` as a dependency. --- Makefile | 2 +- cytoolz/tests/dev_skip_test.py | 8 ++++++++ cytoolz/tests/test_curried_toolzlike.py | 2 +- cytoolz/tests/test_docstrings.py | 3 ++- cytoolz/tests/test_embedded_sigs.py | 2 +- cytoolz/utils.py | 8 -------- 6 files changed, 13 insertions(+), 12 deletions(-) create mode 100644 cytoolz/tests/dev_skip_test.py diff --git a/Makefile b/Makefile index 69a668e..413fc46 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ PYTHON ?= python inplace: - $(PYTHON) setup.py build_ext --inplace + $(PYTHON) setup.py build_ext --inplace --cython test: inplace nosetests -s --with-doctest cytoolz/ diff --git a/cytoolz/tests/dev_skip_test.py b/cytoolz/tests/dev_skip_test.py new file mode 100644 index 0000000..da07cc3 --- /dev/null +++ b/cytoolz/tests/dev_skip_test.py @@ -0,0 +1,8 @@ +import nose.tools +import cytoolz + +# Decorator used to skip tests for developmental versions of CyToolz +if 'dev' in cytoolz.__version__: + dev_skip_test = nose.tools.nottest +else: + dev_skip_test = nose.tools.istest diff --git a/cytoolz/tests/test_curried_toolzlike.py b/cytoolz/tests/test_curried_toolzlike.py index 3019bc2..333e229 100644 --- a/cytoolz/tests/test_curried_toolzlike.py +++ b/cytoolz/tests/test_curried_toolzlike.py @@ -3,7 +3,7 @@ import toolz import toolz.curried import types -from cytoolz.utils import dev_skip_test +from dev_skip_test import dev_skip_test # Note that the tests in this file assume `toolz.curry` is a class, but we diff --git a/cytoolz/tests/test_docstrings.py b/cytoolz/tests/test_docstrings.py index 11623c1..4be663c 100644 --- a/cytoolz/tests/test_docstrings.py +++ b/cytoolz/tests/test_docstrings.py @@ -3,7 +3,8 @@ import toolz from cytoolz import curry, identity, keyfilter, valfilter, merge_with -from cytoolz.utils import raises, dev_skip_test +from cytoolz.utils import raises +from dev_skip_test import dev_skip_test # `cytoolz` functions for which "# doctest: +SKIP" were added. diff --git a/cytoolz/tests/test_embedded_sigs.py b/cytoolz/tests/test_embedded_sigs.py index a52e7e5..d2cd69f 100644 --- a/cytoolz/tests/test_embedded_sigs.py +++ b/cytoolz/tests/test_embedded_sigs.py @@ -4,7 +4,7 @@ from types import BuiltinFunctionType from cytoolz import curry, identity, keyfilter, valfilter, merge_with -from cytoolz.utils import dev_skip_test +from dev_skip_test import dev_skip_test @curry diff --git a/cytoolz/utils.py b/cytoolz/utils.py index 1d04349..2d1f6e7 100644 --- a/cytoolz/utils.py +++ b/cytoolz/utils.py @@ -1,6 +1,5 @@ import doctest import inspect -import nose.tools import os.path import cytoolz @@ -105,10 +104,3 @@ def module_doctest(m, *args, **kwargs): """ fix_module_doctest(m) return doctest.testmod(m, *args, **kwargs).failed == 0 - - -# Decorator used to skip tests for developmental versions of CyToolz -if 'dev' in cytoolz.__version__: - dev_skip_test = nose.tools.nottest -else: - dev_skip_test = nose.tools.istest From 8999c186fd25440cbac545bfb9bdb6acd36fd022 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 3 May 2014 18:13:04 -0400 Subject: [PATCH 003/146] Bump version to '0.6.2dev' --- cytoolz/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cytoolz/_version.py b/cytoolz/_version.py index 7ff0020..f84c9d8 100644 --- a/cytoolz/_version.py +++ b/cytoolz/_version.py @@ -1,2 +1,2 @@ -__version__ = '0.6.1' +__version__ = '0.6.2dev' __toolz_version__ = '0.6.0' From 5feb46d9807e9fb7a71efcb54569906e9cd18b52 Mon Sep 17 00:00:00 2001 From: scoder Date: Sun, 4 May 2014 11:12:52 +0200 Subject: [PATCH 004/146] clean up some unnecessary C-API usages and integer sizes note: integer size fixes which require corresponding changes in itertoolz.pxd --- cytoolz/itertoolz.pyx | 73 +++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 41 deletions(-) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 51f2403..dea509f 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -1,7 +1,7 @@ #cython: embedsignature=True from cpython.dict cimport (PyDict_Contains, PyDict_GetItem, PyDict_New, PyDict_SetItem) -from cpython.exc cimport PyErr_Clear, PyErr_GivenExceptionMatches, PyErr_Occurred +from cpython.exc cimport PyErr_Clear, PyErr_ExceptionMatches, PyErr_GivenExceptionMatches, PyErr_Occurred from cpython.list cimport (PyList_Append, PyList_Check, PyList_GET_ITEM, PyList_GET_SIZE, PyList_New) from cpython.ref cimport PyObject, Py_DECREF, Py_INCREF @@ -112,18 +112,15 @@ cpdef dict groupby(object func, object seq): See Also: ``countby`` """ - cdef dict d + cdef dict d = {} cdef list vals cdef PyObject *obj cdef object item, key - d = PyDict_New() for item in seq: key = func(item) obj = PyDict_GetItem(d, key) if obj is NULL: - vals = PyList_New(0) - PyList_Append(vals, item) - PyDict_SetItem(d, key, vals) + PyDict_SetItem(d, key, [item]) else: PyList_Append(obj, item) return d @@ -131,9 +128,9 @@ cpdef dict groupby(object func, object seq): cdef class _merge_sorted: def __cinit__(self, seqs): - cdef int i + cdef Py_ssize_t i cdef object item, it - self.pq = PyList_New(0) + self.pq = [] self.shortcut = None for i, item in enumerate(seqs): @@ -191,9 +188,9 @@ cdef class _merge_sorted: cdef class _merge_sorted_key: def __cinit__(self, seqs, key): - cdef int i + cdef Py_ssize_t i cdef object item, it, k - self.pq = PyList_New(0) + self.pq = [] self.key = key self.shortcut = None @@ -287,10 +284,10 @@ cdef class interleave: Returns a lazy iterator """ def __cinit__(self, seqs, pass_exceptions=()): - self.iters = PyList_New(0) + self.iters = [] for seq in seqs: PyList_Append(self.iters, iter(seq)) - self.newiters = PyList_New(0) + self.newiters = [] self.pass_exceptions = tuple(pass_exceptions) self.i = 0 self.n = PyList_GET_SIZE(self.iters) @@ -311,7 +308,7 @@ cdef class interleave: if self.n == 0: raise StopIteration self.iters = self.newiters - self.newiters = PyList_New(0) + self.newiters = [] val = PyList_GET_ITEM(self.iters, self.i) self.i += 1 obj = PyIter_Next(val) @@ -330,7 +327,7 @@ cdef class interleave: if self.n == 0: raise StopIteration self.iters = self.newiters - self.newiters = PyList_New(0) + self.newiters = [] val = PyList_GET_ITEM(self.iters, self.i) self.i += 1 obj = PyIter_Next(val) @@ -587,7 +584,7 @@ cpdef object get(object ind, object seq, object default=no_default): See Also: pluck """ - cdef int i + cdef Py_ssize_t i cdef object val cdef tuple result cdef PyObject *obj @@ -606,8 +603,7 @@ cpdef object get(object ind, object seq, object default=no_default): for i, val in enumerate(ind): obj = PyObject_GetItem(seq, val) if obj is NULL: - if not PyErr_GivenExceptionMatches(PyErr_Occurred(), - _get_list_exc): + if not PyErr_ExceptionMatches(_get_list_exc): raise PyErr_Occurred() PyErr_Clear() Py_INCREF(default) @@ -692,10 +688,9 @@ cpdef dict frequencies(object seq): countby groupby """ - cdef dict d + cdef dict d = {} cdef PyObject *obj - cdef int val - d = PyDict_New() + cdef Py_ssize_t val for item in seq: obj = PyDict_GetItem(d, item) if obj is NULL: @@ -708,9 +703,8 @@ cpdef dict frequencies(object seq): ''' Alternative implementation of `frequencies` cpdef dict frequencies(object seq): - cdef dict d + cdef dict d = {} cdef int val - d = PyDict_New() for item in seq: if PyDict_Contains(d, item): val = PyObject_GetItem(d, item) @@ -758,10 +752,9 @@ cpdef dict reduceby(object key, object binop, object seq, object init): ... projects, 0) {'CA': 1200000, 'IL': 2100000} """ - cdef dict d + cdef dict d = {} cdef object item, k, val cdef PyObject *obj - d = PyDict_New() for item in seq: k = key(item) obj = PyDict_GetItem(d, k) @@ -833,8 +826,8 @@ cdef class sliding_window: >>> list(map(mean, sliding_window(2, [1, 2, 3, 4]))) [1.5, 2.5, 3.5] """ - def __cinit__(self, int n, object seq): - cdef int i + def __cinit__(self, Py_ssize_t n, object seq): + cdef Py_ssize_t i self.iterseq = iter(seq) self.prev = PyTuple_New(n) for i in range(1, n): @@ -849,7 +842,7 @@ cdef class sliding_window: def __next__(self): cdef tuple current cdef object item - cdef int i + cdef Py_ssize_t i current = PyTuple_New(self.n) for i in range(1, self.n): item = self.prev[i] @@ -907,7 +900,7 @@ cdef class partition_all: See Also: partition """ - def __cinit__(self, int n, object seq): + def __cinit__(self, Py_ssize_t n, object seq): self.n = n self.iterseq = iter(seq) @@ -917,7 +910,7 @@ cdef class partition_all: def __next__(self): cdef tuple result cdef object item - cdef int i = 0 + cdef Py_ssize_t i = 0 result = PyTuple_New(self.n) for item in self.iterseq: Py_INCREF(item) @@ -946,7 +939,7 @@ cpdef object count(object seq): if iter(seq) is not seq and hasattr(seq, '__len__'): return len(seq) cdef object _ - cdef int i = 0 + cdef Py_ssize_t i = 0 for _ in seq: i += 1 return i @@ -979,8 +972,7 @@ cdef class _pluck_index_default: val = next(self.iterseqs) obj = PyObject_GetItem(val, self.ind) if obj is NULL: - if not PyErr_GivenExceptionMatches(PyErr_Occurred(), - _get_exceptions): + if not PyErr_ExceptionMatches(_get_exceptions): raise PyErr_Occurred() PyErr_Clear() return self.default @@ -988,16 +980,16 @@ cdef class _pluck_index_default: cdef class _pluck_list: - def __cinit__(self, list ind, object seqs): + def __cinit__(self, list ind not None, object seqs): self.ind = ind self.iterseqs = iter(seqs) - self.n = PyList_GET_SIZE(ind) + self.n = len(ind) def __iter__(self): return self def __next__(self): - cdef int i + cdef Py_ssize_t i cdef tuple result cdef object val, seq seq = next(self.iterseqs) @@ -1010,17 +1002,17 @@ cdef class _pluck_list: cdef class _pluck_list_default: - def __cinit__(self, list ind, object seqs, object default): + def __cinit__(self, list ind not None, object seqs, object default): self.ind = ind self.iterseqs = iter(seqs) self.default = default - self.n = PyList_GET_SIZE(ind) + self.n = len(ind) def __iter__(self): return self def __next__(self): - cdef int i + cdef Py_ssize_t i cdef object val, seq cdef tuple result seq = next(self.iterseqs) @@ -1028,8 +1020,7 @@ cdef class _pluck_list_default: for i, val in enumerate(self.ind): obj = PyObject_GetItem(seq, val) if obj is NULL: - if not PyErr_GivenExceptionMatches(PyErr_Occurred(), - _get_list_exc): + if not PyErr_ExceptionMatches(_get_list_exc): raise PyErr_Occurred() PyErr_Clear() Py_INCREF(self.default) @@ -1037,7 +1028,7 @@ cdef class _pluck_list_default: else: val = obj Py_INCREF(val) - PyTuple_SET_ITEM(result, i, val) + PyTuple_SET_ITEM(result, i, val) # TODO: redefine with "PyObject* val" and avoid cast return result From af3eaafe3ddcfd1065ceaa10faef8025b33a1cad Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 5 May 2014 12:18:32 -0400 Subject: [PATCH 005/146] Add test for `dev_skip_test` --- cytoolz/tests/dev_skip_test.py | 6 +++--- cytoolz/tests/test_dev_skip_test.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 cytoolz/tests/test_dev_skip_test.py diff --git a/cytoolz/tests/dev_skip_test.py b/cytoolz/tests/dev_skip_test.py index da07cc3..3546f8a 100644 --- a/cytoolz/tests/dev_skip_test.py +++ b/cytoolz/tests/dev_skip_test.py @@ -1,8 +1,8 @@ -import nose.tools import cytoolz +from nose.tools import nottest, istest # Decorator used to skip tests for developmental versions of CyToolz if 'dev' in cytoolz.__version__: - dev_skip_test = nose.tools.nottest + dev_skip_test = nottest else: - dev_skip_test = nose.tools.istest + dev_skip_test = istest diff --git a/cytoolz/tests/test_dev_skip_test.py b/cytoolz/tests/test_dev_skip_test.py new file mode 100644 index 0000000..0c25271 --- /dev/null +++ b/cytoolz/tests/test_dev_skip_test.py @@ -0,0 +1,21 @@ +from dev_skip_test import istest, nottest, dev_skip_test + +d = {} + + +@istest +def test_passes(): + d['istest'] = True + assert True + + +@nottest +def test_fails(): + d['nottest'] = True + assert False + + +def test_dev_skip_test(): + assert dev_skip_test is istest or dev_skip_test is nottest + assert d.get('istest', False) is True + assert d.get('nottest', False) is False From c65491a36a61f1e00ec15dbb47af90bc25533743 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 5 May 2014 12:58:46 -0400 Subject: [PATCH 006/146] Add scoder to AUTHORS.md. Welcome to CyToolz! --- AUTHORS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AUTHORS.md b/AUTHORS.md index 04bd230..8a305fe 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -5,3 +5,5 @@ Erik Welch [@eriknw](https://github.com/eri Lars Buitinck [@larsmans](http://github.com/larsmans) [Thouis (Ray) Jones](http://people.seas.harvard.edu/~thouis) [@thouis](https://github.com/thouis/) + +scoder [@scoder](https://github.com/scoder/) From 57559f06b86d3d06a24dc61df68df7645ee47579 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Mon, 5 May 2014 20:31:25 +0200 Subject: [PATCH 007/146] use cythonize() instead of old-style distutils hack --- setup.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index e690d48..c3b6db1 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ VERSION = info['__version__'] try: - from Cython.Distutils import build_ext + from Cython.Build import cythonize has_cython = True except ImportError: has_cython = False @@ -50,10 +50,8 @@ if use_cython: suffix = '.pyx' - cmdclass = {'build_ext': build_ext} else: suffix = '.c' - cmdclass = {} ext_modules = [] for modname in ['dicttoolz', 'functoolz', 'itertoolz', @@ -61,13 +59,16 @@ ext_modules.append(Extension('cytoolz.' + modname, ['cytoolz/' + modname + suffix])) +if use_cython: + ext_modules = cythonize(ext_modules) + + if __name__ == '__main__': setup( name='cytoolz', version=VERSION, description=('Cython implementation of Toolz: ' 'High performance functional utilities'), - cmdclass=cmdclass, ext_modules=ext_modules, long_description=(open('README.rst').read() if os.path.exists('README.rst') From 4cb8eb90e69a098c64d76e57f684bebd6f6f09c6 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Mon, 5 May 2014 20:56:58 +0200 Subject: [PATCH 008/146] replace "int" usage by safer "Py_ssize_t" types to avoid unnecessary range limitations --- cytoolz/itertoolz.pxd | 22 +++++++++++----------- cytoolz/itertoolz.pyx | 16 +++++++--------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index ee0279d..910b7ea 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -30,8 +30,8 @@ cdef class interleave: cdef list iters cdef list newiters cdef tuple pass_exceptions - cdef int i - cdef int n + cdef Py_ssize_t i + cdef Py_ssize_t n cdef class _unique_key: @@ -54,13 +54,13 @@ cpdef object isiterable(object x) cpdef object isdistinct(object seq) -cpdef object take(int n, object seq) +cpdef object take(Py_ssize_t n, object seq) -cpdef object drop(int n, object seq) +cpdef object drop(Py_ssize_t n, object seq) -cpdef object take_nth(int n, object seq) +cpdef object take_nth(Py_ssize_t n, object seq) cpdef object first(object seq) @@ -69,7 +69,7 @@ cpdef object first(object seq) cpdef object second(object seq) -cpdef object nth(int n, object seq) +cpdef object nth(Py_ssize_t n, object seq) cpdef object last(object seq) @@ -109,14 +109,14 @@ cdef class iterate: cdef class sliding_window: cdef object iterseq cdef tuple prev - cdef int n + cdef Py_ssize_t n -cpdef object partition(int n, object seq, object pad=*) +cpdef object partition(Py_ssize_t n, object seq, object pad=*) cdef class partition_all: - cdef int n + cdef Py_ssize_t n cdef object iterseq @@ -137,14 +137,14 @@ cdef class _pluck_index_default: cdef class _pluck_list: cdef list ind cdef object iterseqs - cdef int n + cdef Py_ssize_t n cdef class _pluck_list_default: cdef list ind cdef object iterseqs cdef object default - cdef int n + cdef Py_ssize_t n cpdef object pluck(object ind, object seqs, object default=*) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index dea509f..8b2293c 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -439,7 +439,7 @@ cpdef object isdistinct(object seq): return len(seq) == len(set(seq)) -cpdef object take(int n, object seq): +cpdef object take(Py_ssize_t n, object seq): """ The first n elements of a sequence @@ -449,7 +449,7 @@ cpdef object take(int n, object seq): return islice(seq, n) -cpdef object drop(int n, object seq): +cpdef object drop(Py_ssize_t n, object seq): """ The sequence following the first n elements @@ -458,20 +458,18 @@ cpdef object drop(int n, object seq): """ if n < 0: raise ValueError('n argument for drop() must be non-negative') - cdef int i + cdef Py_ssize_t i cdef object iter_seq - i = 0 iter_seq = iter(seq) try: - while i < n: - i += 1 + for i in range(n): next(iter_seq) except StopIteration: pass return iter_seq -cpdef object take_nth(int n, object seq): +cpdef object take_nth(Py_ssize_t n, object seq): """ Every nth item in seq @@ -503,7 +501,7 @@ cpdef object second(object seq): return next(seq) -cpdef object nth(int n, object seq): +cpdef object nth(Py_ssize_t n, object seq): """ The nth element in a sequence @@ -858,7 +856,7 @@ cdef class sliding_window: no_pad = '__no__pad__' -cpdef object partition(int n, object seq, object pad=no_pad): +cpdef object partition(Py_ssize_t n, object seq, object pad=no_pad): """ Partition sequence into tuples of length n From 4df68b1b59b43a260ed887f849d8a7b2a9e9e92b Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Mon, 5 May 2014 21:56:31 +0200 Subject: [PATCH 009/146] remove some unnecessary C-API usages - list comprehensions are actually faster than repeated calls to PyList_Append() - Py_XDECREF() works nicely on PyObject* (and the C compiler will usually drop the NULL check) - isinstance(x, builtin) translates directly to a PyBuiltin_Check() call --- cytoolz/itertoolz.pyx | 51 +++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index dea509f..4617ebc 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -1,10 +1,9 @@ #cython: embedsignature=True -from cpython.dict cimport (PyDict_Contains, PyDict_GetItem, PyDict_New, - PyDict_SetItem) -from cpython.exc cimport PyErr_Clear, PyErr_ExceptionMatches, PyErr_GivenExceptionMatches, PyErr_Occurred -from cpython.list cimport (PyList_Append, PyList_Check, PyList_GET_ITEM, - PyList_GET_SIZE, PyList_New) -from cpython.ref cimport PyObject, Py_DECREF, Py_INCREF +from cpython.dict cimport PyDict_GetItem +from cpython.exc cimport (PyErr_Clear, PyErr_ExceptionMatches, + PyErr_GivenExceptionMatches, PyErr_Occurred) +from cpython.list cimport (PyList_Append, PyList_GET_ITEM, PyList_GET_SIZE) +from cpython.ref cimport PyObject, Py_DECREF, Py_INCREF, Py_XDECREF from cpython.sequence cimport PySequence_Check from cpython.set cimport PySet_Add, PySet_Contains from cpython.tuple cimport PyTuple_GetSlice, PyTuple_New, PyTuple_SET_ITEM @@ -120,7 +119,7 @@ cpdef dict groupby(object func, object seq): key = func(item) obj = PyDict_GetItem(d, key) if obj is NULL: - PyDict_SetItem(d, key, [item]) + d[key] = [item] else: PyList_Append(obj, item) return d @@ -263,7 +262,7 @@ def merge_sorted(*seqs, **kwargs): >>> list(merge_sorted([2, 3], [1, 3], key=lambda x: x // 3)) [2, 1, 3, 3] """ - if PyDict_Contains(kwargs, 'key'): + if 'key' in kwargs: return c_merge_sorted(seqs, kwargs['key']) return c_merge_sorted(seqs) @@ -284,9 +283,7 @@ cdef class interleave: Returns a lazy iterator """ def __cinit__(self, seqs, pass_exceptions=()): - self.iters = [] - for seq in seqs: - PyList_Append(self.iters, iter(seq)) + self.iters = [iter(seq) for seq in seqs] self.newiters = [] self.pass_exceptions = tuple(pass_exceptions) self.i = 0 @@ -334,7 +331,7 @@ cdef class interleave: PyList_Append(self.newiters, val) val = obj - Py_DECREF(val) + Py_XDECREF(obj) return val @@ -433,7 +430,7 @@ cpdef object isdistinct(object seq): for item in seq: if PySet_Contains(seen, item): return False - PySet_Add(seen, item) + seen.add(item) return True else: return len(seq) == len(set(seq)) @@ -588,7 +585,7 @@ cpdef object get(object ind, object seq, object default=no_default): cdef object val cdef tuple result cdef PyObject *obj - if PyList_Check(ind): + if isinstance(ind, list): i = PyList_GET_SIZE(ind) result = PyTuple_New(i) # List of indices, no default @@ -694,23 +691,23 @@ cpdef dict frequencies(object seq): for item in seq: obj = PyDict_GetItem(d, item) if obj is NULL: - PyDict_SetItem(d, item, 1) + d[item] = 1 else: val = obj - PyDict_SetItem(d, item, val + 1) + d[item] = val + 1 return d ''' Alternative implementation of `frequencies` cpdef dict frequencies(object seq): cdef dict d = {} - cdef int val + cdef Py_ssize_t val for item in seq: - if PyDict_Contains(d, item): - val = PyObject_GetItem(d, item) - PyDict_SetItem(d, item, val + 1) + if item in d: + val = d[item] + d[item] = val + 1 else: - PyDict_SetItem(d, item, 1) + d[item] = 1 return d ''' @@ -763,7 +760,7 @@ cpdef dict reduceby(object key, object binop, object seq, object init): else: val = obj val = binop(val, item) - PyDict_SetItem(d, k, val) + d[k] = val return d @@ -917,12 +914,11 @@ cdef class partition_all: PyTuple_SET_ITEM(result, i, item) i += 1 if i == self.n: - break + return result + # iterable exhausted before filling the tuple if i == 0: raise StopIteration - if i != self.n: - return PyTuple_GetSlice(result, 0, i) - return result + return PyTuple_GetSlice(result, 0, i) cpdef object count(object seq): @@ -938,7 +934,6 @@ cpdef object count(object seq): """ if iter(seq) is not seq and hasattr(seq, '__len__'): return len(seq) - cdef object _ cdef Py_ssize_t i = 0 for _ in seq: i += 1 @@ -1057,7 +1052,7 @@ cpdef object pluck(object ind, object seqs, object default=no_default): get map """ - if PyList_Check(ind): + if isinstance(ind, list): if default is not no_default: return _pluck_list_default(ind, seqs, default) if PyList_GET_SIZE(ind) < 10: From dd2d95f38d794b371f0e2be11a4b38148ce9ba28 Mon Sep 17 00:00:00 2001 From: Matthew Rocklin Date: Thu, 5 Jun 2014 16:40:27 -0700 Subject: [PATCH 010/146] add join This is ugly and likely inefficient. Help. --- cytoolz/itertoolz.pxd | 17 ++++ cytoolz/itertoolz.pyx | 135 ++++++++++++++++++++++++++++++++ cytoolz/tests/test_itertoolz.py | 95 +++++++++++++++++++++- 3 files changed, 246 insertions(+), 1 deletion(-) diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index 910b7ea..0bd44e5 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -148,3 +148,20 @@ cdef class _pluck_list_default: cpdef object pluck(object ind, object seqs, object default=*) + +cdef class join: + cdef Py_ssize_t n + cdef object iterseq + cdef object leftkey + cdef object leftseq + cdef object rightkey + cdef object rightseq + cdef object matches + cdef object right + cdef object key + cdef object d + cdef object d_items + cdef object seen_keys + cdef object is_rightseq_exhausted + cdef object left_default + cdef object right_default diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index de1b598..b90303d 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -1061,6 +1061,141 @@ cpdef object pluck(object ind, object seqs, object default=no_default): return _pluck_index_default(ind, seqs, default) +def getter(index): + if isinstance(index, list): + if len(index) == 1: + index = index[0] + return lambda x: (x[index],) + else: + return itemgetter(*index) + else: + return itemgetter(index) + + +cdef class join: + """ Join two sequences on common attributes + + This is a semi-streaming operation. The LEFT sequence is fully evaluated + and placed into memory. The RIGHT sequence is evaluated lazily and so can + be arbitrarily large. + + >>> friends = [('Alice', 'Edith'), + ... ('Alice', 'Zhao'), + ... ('Edith', 'Alice'), + ... ('Zhao', 'Alice'), + ... ('Zhao', 'Edith')] + + >>> cities = [('Alice', 'NYC'), + ... ('Alice', 'Chicago'), + ... ('Dan', 'Syndey'), + ... ('Edith', 'Paris'), + ... ('Edith', 'Berlin'), + ... ('Zhao', 'Shanghai')] + + >>> # Vacation opportunities + >>> # In what cities do people have friends? + >>> result = join(second, friends, + ... first, cities) + >>> for ((a, b), (c, d)) in sorted(unique(result)): + ... print((a, d)) + ('Alice', 'Berlin') + ('Alice', 'Paris') + ('Alice', 'Shanghai') + ('Edith', 'Chicago') + ('Edith', 'NYC') + ('Zhao', 'Chicago') + ('Zhao', 'NYC') + ('Zhao', 'Berlin') + ('Zhao', 'Paris') + + Specify outer joins with keyword arguments ``left_default`` and/or + ``right_default``. Here is a full outer join in which unmatched elements + are paired with None. + + >>> identity = lambda x: x + >>> list(join(identity, [1, 2, 3], + ... identity, [2, 3, 4], + ... left_default=None, right_default=None)) + [(2, 2), (3, 3), (None, 4), (1, None)] + + Usually the key arguments are callables to be applied to the sequences. If + the keys are not obviously callable then it is assumed that indexing was + intended, e.g. the following is a legal change + + >>> # result = join(second, friends, first, cities) + >>> result = join(1, friends, 0, cities) # doctest: +SKIP + """ + def __init__(self, + object leftkey, object leftseq, + object rightkey, object rightseq, + object left_default=no_default, + object right_default=no_default): + if not callable(leftkey): + leftkey = getter(leftkey) + if not callable(rightkey): + rightkey = getter(rightkey) + + self.left_default = left_default + self.right_default = right_default + + self.leftkey = leftkey + self.rightkey = rightkey + self.rightseq = iter(rightseq) + + self.d = groupby(leftkey, leftseq) + self.seen_keys = set() + self.matches = iter(()) + self.right = None + + self.is_rightseq_exhausted = False + + + def __iter__(self): + return self + + def __next__(self): + if not self.is_rightseq_exhausted: + try: + match = next(self.matches) + return (match, self.right) + except StopIteration: # iterator of matches exhausted + try: + item = next(self.rightseq) # get a new item + except StopIteration: # no items, switch to outer join + self.is_rightseq_exhausted = True + if self.right_default is not no_default: + self.d_items = iter(self.d.items()) + self.matches = iter(()) + return next(self) + else: + raise + + key = self.rightkey(item) + self.seen_keys.add(key) + + try: + self.matches = iter(self.d[key]) # get left matches + except KeyError: + if self.left_default is not no_default: + return (self.left_default, item) # outer join + + self.right = item + return next(self) + + else: # we've exhausted the right sequence, lets iterate over unseen + # items on the left + try: + match = next(self.matches) + return (match, self.right_default) + except StopIteration: + key, matches = next(self.d_items) + while(key in self.seen_keys and matches): + key, matches = next(self.d_items) + self.key = key + self.matches = iter(matches) + return next(self) + + # I find `_consume` convenient for benchmarking. Perhaps this belongs # elsewhere, so it is private (leading underscore) and hidden away for now. diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index a3404ff..7a16123 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -1,4 +1,5 @@ import itertools +from itertools import starmap from cytoolz.utils import raises from functools import partial from cytoolz.itertoolz import (remove, groupby, merge_sorted, @@ -9,7 +10,7 @@ rest, last, cons, frequencies, reduceby, iterate, accumulate, sliding_window, count, partition, - partition_all, take_nth, pluck) + partition_all, take_nth, pluck, join) from cytoolz.compatibility import range, filter from operator import add, mul @@ -264,3 +265,95 @@ def test_pluck(): assert raises(IndexError, lambda: list(pluck(1, [[0]]))) assert raises(KeyError, lambda: list(pluck('name', [{'id': 1}]))) + + +def test_join(): + names = [(1, 'one'), (2, 'two'), (3, 'three')] + fruit = [('apple', 1), ('orange', 1), ('banana', 2), ('coconut', 2)] + + def addpair(pair): + return pair[0] + pair[1] + + result = set(starmap(add, join(first, names, second, fruit))) + + expected = set([((1, 'one', 'apple', 1)), + ((1, 'one', 'orange', 1)), + ((2, 'two', 'banana', 2)), + ((2, 'two', 'coconut', 2))]) + + print(result) + print(expected) + assert result == expected + + +def test_key_as_getter(): + squares = [(i, i**2) for i in range(5)] + pows = [(i, i**2, i**3) for i in range(5)] + + assert set(join(0, squares, 0, pows)) == set(join(lambda x: x[0], squares, + lambda x: x[0], pows)) + + get = lambda x: (x[0], x[1]) + assert set(join([0, 1], squares, [0, 1], pows)) == set(join(get, squares, + get, pows)) + + get = lambda x: (x[0],) + assert set(join([0], squares, [0], pows)) == set(join(get, squares, + get, pows)) + + +def test_join_double_repeats(): + names = [(1, 'one'), (2, 'two'), (3, 'three'), (1, 'uno'), (2, 'dos')] + fruit = [('apple', 1), ('orange', 1), ('banana', 2), ('coconut', 2)] + + result = set(starmap(add, join(first, names, second, fruit))) + + expected = set([((1, 'one', 'apple', 1)), + ((1, 'one', 'orange', 1)), + ((2, 'two', 'banana', 2)), + ((2, 'two', 'coconut', 2)), + ((1, 'uno', 'apple', 1)), + ((1, 'uno', 'orange', 1)), + ((2, 'dos', 'banana', 2)), + ((2, 'dos', 'coconut', 2))]) + + print(result) + print(expected) + assert result == expected + + +def test_join_missing_element(): + names = [(1, 'one'), (2, 'two'), (3, 'three')] + fruit = [('apple', 5), ('orange', 1)] + + result = list(join(first, names, second, fruit)) + print(result) + result = set(starmap(add, result)) + + expected = set([((1, 'one', 'orange', 1))]) + + assert result == expected + + +def test_left_outer_join(): + result = set(join(identity, [1, 2], identity, [2, 3], left_default=None)) + expected = set([(2, 2), (None, 3)]) + + print(result) + print(expected) + assert result == expected + + +def test_right_outer_join(): + result = set(join(identity, [1, 2], identity, [2, 3], right_default=None)) + expected = set([(2, 2), (1, None)]) + + assert result == expected + + +def test_outer_join(): + result = set(join(identity, [1, 2], identity, [2, 3], + left_default=None, right_default=None)) + expected = set([(2, 2), (1, None), (None, 3)]) + + assert result == expected From 919a97ca92d10869850df5939eff4d2f44076e97 Mon Sep 17 00:00:00 2001 From: Matthew Rocklin Date: Sun, 8 Jun 2014 07:37:50 -0700 Subject: [PATCH 011/146] split out different joins into different classes I'm not sure that this is worth it --- cytoolz/itertoolz.pxd | 21 ++++++- cytoolz/itertoolz.pyx | 139 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 156 insertions(+), 4 deletions(-) diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index 0bd44e5..99ec432 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -149,7 +149,12 @@ cdef class _pluck_list_default: cpdef object pluck(object ind, object seqs, object default=*) -cdef class join: +cpdef object join(object leftkey, object leftseq, + object rightkey, object rightseq, + object left_default=*, + object right_default=*) + +cdef class _join: cdef Py_ssize_t n cdef object iterseq cdef object leftkey @@ -165,3 +170,17 @@ cdef class join: cdef object is_rightseq_exhausted cdef object left_default cdef object right_default + +cdef class _inner_join(_join): + cdef int i + +cdef class _right_outer_join(_join): + cdef int i + +cdef class _left_outer_join(_join): + cdef int i + cdef object keys + +cdef class _outer_join(_join): + cdef int i + cdef object keys diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index b90303d..0f40ac3 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -1071,8 +1071,10 @@ def getter(index): else: return itemgetter(index) - -cdef class join: +cpdef object join(object leftkey, object leftseq, + object rightkey, object rightseq, + object left_default=no_default, + object right_default=no_default): """ Join two sequences on common attributes This is a semi-streaming operation. The LEFT sequence is fully evaluated @@ -1125,6 +1127,20 @@ cdef class join: >>> # result = join(second, friends, first, cities) >>> result = join(1, friends, 0, cities) # doctest: +SKIP """ + if left_default == no_default and right_default == no_default: + return _inner_join(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) + elif left_default != no_default and right_default == no_default: + return _right_outer_join(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) + elif left_default == no_default and right_default != no_default: + return _left_outer_join(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) + else: + return _outer_join(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) + +cdef class _join: def __init__(self, object leftkey, object leftseq, object rightkey, object rightseq, @@ -1172,7 +1188,6 @@ cdef class join: key = self.rightkey(item) self.seen_keys.add(key) - try: self.matches = iter(self.d[key]) # get left matches except KeyError: @@ -1196,6 +1211,124 @@ cdef class join: return next(self) +cdef class _right_outer_join(_join): + def __iter__(self): + self.matches = () + return self + + def __next__(self): + cdef PyObject *obj + if self.i == len(self.matches): + self.right = next(self.rightseq) + key = self.rightkey(self.right) + obj = PyDict_GetItem(self.d, key) + if obj is NULL: + return (self.left_default, self.right) + self.matches = obj + self.i = 0 + match = PyList_GET_ITEM(self.matches, self.i) # skip error checking + self.i += 1 + return (match, self.right) + + +cdef class _outer_join(_join): + def __iter__(self): + self.matches = () + return self + + def __next__(self): + cdef PyObject *obj + if not self.is_rightseq_exhausted: + if self.i == len(self.matches): + try: + self.right = next(self.rightseq) + except StopIteration: + self.is_rightseq_exhausted = True + self.keys = iter(self.d) + return next(self) + key = self.rightkey(self.right) + self.seen_keys.add(key) + obj = PyDict_GetItem(self.d, key) + if obj is NULL: + return (self.left_default, self.right) + self.matches = obj + self.i = 0 + match = PyList_GET_ITEM(self.matches, self.i) # skip error checking + self.i += 1 + return (match, self.right) + + else: + if self.i == len(self.matches): + key = next(self.keys) + while key in self.seen_keys: + key = next(self.keys) + obj = PyDict_GetItem(self.d, key) + self.matches = obj + self.i = 0 + match = PyList_GET_ITEM(self.matches, self.i) # skip error checking + self.i += 1 + return (match, self.right_default) + + + +cdef class _left_outer_join(_join): + def __iter__(self): + self.matches = () + return self + + def __next__(self): + cdef PyObject *obj + if not self.is_rightseq_exhausted: + if self.i == len(self.matches): + obj = NULL + while obj is NULL: + try: + self.right = next(self.rightseq) + except StopIteration: + self.is_rightseq_exhausted = True + self.keys = iter(self.d) + return next(self) + key = self.rightkey(self.right) + self.seen_keys.add(key) + obj = PyDict_GetItem(self.d, key) + self.matches = obj + self.i = 0 + match = PyList_GET_ITEM(self.matches, self.i) # skip error checking + self.i += 1 + return (match, self.right) + + else: + if self.i == len(self.matches): + key = next(self.keys) + while key in self.seen_keys: + key = next(self.keys) + obj = PyDict_GetItem(self.d, key) + self.matches = obj + self.i = 0 + match = PyList_GET_ITEM(self.matches, self.i) # skip error checking + self.i += 1 + return (match, self.right_default) + + +cdef class _inner_join(_join): + def __iter__(self): + self.matches = () + return self + + def __next__(self): + cdef PyObject *obj = NULL + if self.i == len(self.matches): + while obj is NULL: + self.right = next(self.rightseq) + key = self.rightkey(self.right) + obj = PyDict_GetItem(self.d, key) + self.matches = obj + self.i = 0 + match = PyList_GET_ITEM(self.matches, self.i) # skip error checking + self.i += 1 + return (match, self.right) + + # I find `_consume` convenient for benchmarking. Perhaps this belongs # elsewhere, so it is private (leading underscore) and hidden away for now. From 1734d7c01cb310a81e2ccd25ac52cc563dec7e68 Mon Sep 17 00:00:00 2001 From: Matthew Rocklin Date: Sun, 8 Jun 2014 07:55:32 -0700 Subject: [PATCH 012/146] single join class with single __next__ --- cytoolz/itertoolz.pxd | 12 +++---- cytoolz/itertoolz.pyx | 79 ++++++++++++++++++------------------------- 2 files changed, 39 insertions(+), 52 deletions(-) diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index 99ec432..be4798b 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -170,17 +170,17 @@ cdef class _join: cdef object is_rightseq_exhausted cdef object left_default cdef object right_default + cdef int i + cdef object keys cdef class _inner_join(_join): - cdef int i + pass cdef class _right_outer_join(_join): - cdef int i + pass cdef class _left_outer_join(_join): - cdef int i - cdef object keys + pass cdef class _outer_join(_join): - cdef int i - cdef object keys + pass diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 0f40ac3..ef07e6f 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -1127,6 +1127,8 @@ cpdef object join(object leftkey, object leftseq, >>> # result = join(second, friends, first, cities) >>> result = join(1, friends, 0, cities) # doctest: +SKIP """ + return _join(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) if left_default == no_default and right_default == no_default: return _inner_join(leftkey, leftseq, rightkey, rightseq, left_default, right_default) @@ -1160,7 +1162,7 @@ cdef class _join: self.d = groupby(leftkey, leftseq) self.seen_keys = set() - self.matches = iter(()) + self.matches = () self.right = None self.is_rightseq_exhausted = False @@ -1170,52 +1172,45 @@ cdef class _join: return self def __next__(self): + cdef PyObject *obj if not self.is_rightseq_exhausted: - try: - match = next(self.matches) - return (match, self.right) - except StopIteration: # iterator of matches exhausted + if self.i == len(self.matches): try: - item = next(self.rightseq) # get a new item - except StopIteration: # no items, switch to outer join - self.is_rightseq_exhausted = True - if self.right_default is not no_default: - self.d_items = iter(self.d.items()) - self.matches = iter(()) - return next(self) - else: + self.right = next(self.rightseq) + except StopIteration: + if self.right_default is no_default: raise - - key = self.rightkey(item) + self.is_rightseq_exhausted = True + self.keys = iter(self.d) + return next(self) + key = self.rightkey(self.right) self.seen_keys.add(key) - try: - self.matches = iter(self.d[key]) # get left matches - except KeyError: + obj = PyDict_GetItem(self.d, key) + if obj is NULL: if self.left_default is not no_default: - return (self.left_default, item) # outer join - - self.right = item - return next(self) + return (self.left_default, self.right) + else: + return next(self) + self.matches = obj + self.i = 0 + match = PyList_GET_ITEM(self.matches, self.i) # skip error checking + self.i += 1 + return (match, self.right) - else: # we've exhausted the right sequence, lets iterate over unseen - # items on the left - try: - match = next(self.matches) - return (match, self.right_default) - except StopIteration: - key, matches = next(self.d_items) - while(key in self.seen_keys and matches): - key, matches = next(self.d_items) - self.key = key - self.matches = iter(matches) - return next(self) + elif self.right_default is not no_default: + if self.i == len(self.matches): + key = next(self.keys) + while key in self.seen_keys: + key = next(self.keys) + obj = PyDict_GetItem(self.d, key) + self.matches = obj + self.i = 0 + match = PyList_GET_ITEM(self.matches, self.i) # skip error checking + self.i += 1 + return (match, self.right_default) cdef class _right_outer_join(_join): - def __iter__(self): - self.matches = () - return self - def __next__(self): cdef PyObject *obj if self.i == len(self.matches): @@ -1232,10 +1227,6 @@ cdef class _right_outer_join(_join): cdef class _outer_join(_join): - def __iter__(self): - self.matches = () - return self - def __next__(self): cdef PyObject *obj if not self.is_rightseq_exhausted: @@ -1272,10 +1263,6 @@ cdef class _outer_join(_join): cdef class _left_outer_join(_join): - def __iter__(self): - self.matches = () - return self - def __next__(self): cdef PyObject *obj if not self.is_rightseq_exhausted: From 78f725ae4b539954695706852949e318fee41085 Mon Sep 17 00:00:00 2001 From: Matthew Rocklin Date: Sun, 8 Jun 2014 10:35:12 -0700 Subject: [PATCH 013/146] remove loose __iter__ --- cytoolz/itertoolz.pyx | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index ef07e6f..2382087 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -1298,10 +1298,6 @@ cdef class _left_outer_join(_join): cdef class _inner_join(_join): - def __iter__(self): - self.matches = () - return self - def __next__(self): cdef PyObject *obj = NULL if self.i == len(self.matches): From 5867a32c3ea9cfc4155a7efb2d0b3ead349fb72a Mon Sep 17 00:00:00 2001 From: Matthew Rocklin Date: Mon, 9 Jun 2014 19:50:09 -0700 Subject: [PATCH 014/146] add join to __init__.pxd --- cytoolz/__init__.pxd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cytoolz/__init__.pxd b/cytoolz/__init__.pxd index 4468950..7e08db8 100644 --- a/cytoolz/__init__.pxd +++ b/cytoolz/__init__.pxd @@ -2,7 +2,7 @@ from cytoolz.itertoolz cimport ( accumulate, c_merge_sorted, cons, count, drop, get, groupby, first, frequencies, interleave, interpose, isdistinct, isiterable, iterate, last, mapcat, nth, partition, partition_all, pluck, reduceby, remove, - rest, second, sliding_window, take, take_nth, unique) + rest, second, sliding_window, take, take_nth, unique, join) from cytoolz.functoolz cimport ( From ce92e0bb013df81bb9645f8949cf2b1c537746e7 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 23 Jun 2014 13:03:50 -0400 Subject: [PATCH 015/146] Rename additional C bindings defined in `cytoolz.cpython` This differentiates the additional bindings with a common convention. Currently, we only change bindings to return pointers instead of Python objects, and we change the name from "Py*" to "Ptr*". I expect we will add more bindings in the future, including passing pointers to functions. In this case, the name should be something like "*_ptr". --- cytoolz/cpython.pxd | 6 +++--- cytoolz/dicttoolz.pyx | 4 ++-- cytoolz/functoolz.pyx | 4 ++-- cytoolz/itertoolz.pyx | 14 +++++++------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cytoolz/cpython.pxd b/cytoolz/cpython.pxd index 4c3e244..7dfca1c 100644 --- a/cytoolz/cpython.pxd +++ b/cytoolz/cpython.pxd @@ -5,6 +5,6 @@ These differ from Cython's bindings in ``cpython``. from cpython.ref cimport PyObject cdef extern from "Python.h": - PyObject* PyIter_Next(object o) - PyObject* PyObject_Call(object callable_object, object args, object kw) - PyObject* PyObject_GetItem(object o, object key) + PyObject* PtrIter_Next "PyIter_Next"(object o) + PyObject* PtrObject_Call "PyObject_Call"(object callable_object, object args, object kw) + PyObject* PtrObject_GetItem "PyObject_GetItem"(object o, object key) diff --git a/cytoolz/dicttoolz.pyx b/cytoolz/dicttoolz.pyx index 7c69e93..0b576eb 100644 --- a/cytoolz/dicttoolz.pyx +++ b/cytoolz/dicttoolz.pyx @@ -7,7 +7,7 @@ from cpython.list cimport PyList_Append, PyList_New from cpython.ref cimport PyObject # Locally defined bindings that differ from `cython.cpython` bindings -from .cpython cimport PyObject_GetItem +from .cpython cimport PtrObject_GetItem __all__ = ['merge', 'merge_with', 'valmap', 'keymap', 'valfilter', 'keyfilter', @@ -316,7 +316,7 @@ cpdef object get_in(object keys, object coll, object default=None, object no_def cdef object item cdef PyObject *obj for item in keys: - obj = PyObject_GetItem(coll, item) + obj = PtrObject_GetItem(coll, item) if obj is NULL: item = PyErr_Occurred() if no_default or not PyErr_GivenExceptionMatches(item, _get_in_exceptions): diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 48d85a4..464cf21 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -9,7 +9,7 @@ from cpython.set cimport PyFrozenSet_New from cpython.tuple cimport PyTuple_Check, PyTuple_GET_SIZE # Locally defined bindings that differ from `cython.cpython` bindings -from .cpython cimport PyObject_Call as CyObject_Call +from .cpython cimport PtrObject_Call __all__ = ['identity', 'thread_first', 'thread_last', 'memoize', 'compose', @@ -211,7 +211,7 @@ cdef class curry: if self.keywords is not None: PyDict_Merge(kwargs, self.keywords, False) - obj = CyObject_Call(self.func, args, kwargs) + obj = PtrObject_Call(self.func, args, kwargs) if obj is not NULL: val = obj return val diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index de1b598..22fa088 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -9,7 +9,7 @@ from cpython.set cimport PySet_Add, PySet_Contains from cpython.tuple cimport PyTuple_GetSlice, PyTuple_New, PyTuple_SET_ITEM # Locally defined bindings that differ from `cython.cpython` bindings -from .cpython cimport PyIter_Next, PyObject_GetItem +from .cpython cimport PtrIter_Next, PtrObject_GetItem from heapq import heapify, heappop, heapreplace from itertools import chain, islice @@ -308,7 +308,7 @@ cdef class interleave: self.newiters = [] val = PyList_GET_ITEM(self.iters, self.i) self.i += 1 - obj = PyIter_Next(val) + obj = PtrIter_Next(val) while obj is NULL: obj = PyErr_Occurred() @@ -327,7 +327,7 @@ cdef class interleave: self.newiters = [] val = PyList_GET_ITEM(self.iters, self.i) self.i += 1 - obj = PyIter_Next(val) + obj = PtrIter_Next(val) PyList_Append(self.newiters, val) val = obj @@ -596,7 +596,7 @@ cpdef object get(object ind, object seq, object default=no_default): # List of indices with default for i, val in enumerate(ind): - obj = PyObject_GetItem(seq, val) + obj = PtrObject_GetItem(seq, val) if obj is NULL: if not PyErr_ExceptionMatches(_get_list_exc): raise PyErr_Occurred() @@ -609,7 +609,7 @@ cpdef object get(object ind, object seq, object default=no_default): PyTuple_SET_ITEM(result, i, val) return result - obj = PyObject_GetItem(seq, ind) + obj = PtrObject_GetItem(seq, ind) if obj is NULL: val = PyErr_Occurred() if default is no_default: @@ -963,7 +963,7 @@ cdef class _pluck_index_default: cdef PyObject *obj cdef object val val = next(self.iterseqs) - obj = PyObject_GetItem(val, self.ind) + obj = PtrObject_GetItem(val, self.ind) if obj is NULL: if not PyErr_ExceptionMatches(_get_exceptions): raise PyErr_Occurred() @@ -1011,7 +1011,7 @@ cdef class _pluck_list_default: seq = next(self.iterseqs) result = PyTuple_New(self.n) for i, val in enumerate(self.ind): - obj = PyObject_GetItem(seq, val) + obj = PtrObject_GetItem(seq, val) if obj is NULL: if not PyErr_ExceptionMatches(_get_list_exc): raise PyErr_Occurred() From b349fb8ead1ba3c19cf99bfb27fb2f87d0571756 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 23 Jun 2014 13:16:13 -0400 Subject: [PATCH 016/146] Simplify TravisCI script, which makes travis finish faster I test installation with the C files that *I* create using this branch: https://github.com/eriknw/cytoolz/tree/check_my_c_files Hence, we don't need to verify that cytoolz can be installed with only a C compiler for every commit. --- .travis.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2fd5f8e..66df268 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,21 +5,12 @@ python: - "3.2" - "3.3" -env: - - WITH_CYTHON=true - - WITH_CYTHON=false - before_install: - pip install git+https://github.com/pytoolz/toolz.git - pip install cython install: - # *.c files aren't in the repo, so we must always create them - python setup.py build_ext --inplace --with-cython - - if [[ $WITH_CYTHON == 'false' ]]; then - - rm cytoolz/*.so - - python setup.py build_ext --inplace --without-cython - - fi # commands to run tests script: From cf8d9c2c19b74e25d483838378f42ee7468cfa6f Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 23 Jun 2014 20:27:52 -0400 Subject: [PATCH 017/146] Support serialization of curry, compose, juxt, and complement objects. --- cytoolz/functoolz.pxd | 2 +- cytoolz/functoolz.pyx | 22 +++++++++++++++++++-- cytoolz/tests/test_serialization.py | 30 +++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 cytoolz/tests/test_serialization.py diff --git a/cytoolz/functoolz.pxd b/cytoolz/functoolz.pxd index 070120d..4bf4f2e 100644 --- a/cytoolz/functoolz.pxd +++ b/cytoolz/functoolz.pxd @@ -40,7 +40,7 @@ cdef class complement: cdef class _juxt_inner: - cdef tuple funcs + cdef public tuple funcs cdef object c_juxt(object funcs) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 48d85a4..258caae 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -225,6 +225,12 @@ cdef class curry: return curry(self.func, *args, **kwargs) raise val + def __reduce__(self): + return (curry, (self.func,), (self.args, self.keywords)) + + def __setstate__(self, state): + self.args, self.keywords = state + cdef class c_memoize: property __doc__: @@ -336,6 +342,12 @@ cdef class Compose: ret = func(ret) return ret + def __reduce__(self): + return (Compose, (self.firstfunc,), self.funcs) + + def __setstate__(self, state): + self.funcs = state + cdef object c_compose(object funcs): if not funcs: @@ -418,6 +430,9 @@ cdef class complement: def __call__(self, *args, **kwargs): return not PyObject_Call(self.func, args, kwargs) # use PyObject_Not? + def __reduce__(self): + return (complement, (self.func,)) + cdef class _juxt_inner: def __cinit__(self, funcs): @@ -425,9 +440,12 @@ cdef class _juxt_inner: def __call__(self, *args, **kwargs): if kwargs: - return (PyObject_Call(func, args, kwargs) for func in self.funcs) + return tuple(PyObject_Call(func, args, kwargs) for func in self.funcs) else: - return (PyObject_CallObject(func, args) for func in self.funcs) + return tuple(PyObject_CallObject(func, args) for func in self.funcs) + + def __reduce__(self): + return (_juxt_inner, (self.funcs,)) cdef object c_juxt(object funcs): diff --git a/cytoolz/tests/test_serialization.py b/cytoolz/tests/test_serialization.py new file mode 100644 index 0000000..9d2464f --- /dev/null +++ b/cytoolz/tests/test_serialization.py @@ -0,0 +1,30 @@ +from cytoolz import * +import pickle + + +def test_compose(): + f = compose(str, sum) + g = pickle.loads(pickle.dumps(f)) + assert f((1, 2)) == g((1, 2)) + + +def test_curry(): + f = curry(map)(str) + g = pickle.loads(pickle.dumps(f)) + assert list(f((1, 2, 3))) == list(g((1, 2, 3))) + + +def test_juxt(): + f = juxt(str, int, bool) + g = pickle.loads(pickle.dumps(f)) + assert f(1) == g(1) + assert f.funcs == g.funcs + + +def test_complement(): + f = complement(bool) + assert f(True) is False + assert f(False) is True + g = pickle.loads(pickle.dumps(f)) + assert f(True) == g(True) + assert f(False) == g(False) From 807a44278cf1165639dc2457606074ccd00ee016 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 24 Jun 2014 10:49:34 -0400 Subject: [PATCH 018/146] Update `curry` and `reduceby` to match those in toolz Matches curry changes from pytoolz/toolz#170 Matches reduceby changes from pytoolz/toolz#184 --- cytoolz/functoolz.pxd | 3 +- cytoolz/functoolz.pyx | 32 ++++++---- cytoolz/itertoolz.pxd | 2 +- cytoolz/itertoolz.pyx | 8 ++- cytoolz/tests/test_curried.py | 1 - cytoolz/tests/test_dicttoolz.py | 3 +- cytoolz/tests/test_functoolz.py | 100 ++++++++++++++++++++++++++++++-- cytoolz/tests/test_itertoolz.py | 31 +++++++--- 8 files changed, 148 insertions(+), 32 deletions(-) diff --git a/cytoolz/functoolz.pxd b/cytoolz/functoolz.pxd index 4bf4f2e..b15db7d 100644 --- a/cytoolz/functoolz.pxd +++ b/cytoolz/functoolz.pxd @@ -11,7 +11,8 @@ cdef class curry: cdef readonly object func cdef readonly tuple args cdef readonly dict keywords - + cdef public object __doc__ + cdef public object __name__ cdef class c_memoize: cdef object func diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 2e82fda..d17fe78 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -2,7 +2,8 @@ import inspect from cpython.dict cimport PyDict_Merge, PyDict_New from cpython.exc cimport PyErr_Clear, PyErr_ExceptionMatches, PyErr_Occurred -from cpython.object cimport PyCallable_Check, PyObject_Call, PyObject_CallObject +from cpython.object cimport (PyCallable_Check, PyObject_Call, + PyObject_CallObject, PyObject_RichCompare) from cpython.ref cimport PyObject from cpython.sequence cimport PySequence_Concat from cpython.set cimport PyFrozenSet_New @@ -166,14 +167,6 @@ cdef class curry: cytoolz.curried - namespace of curried functions http://toolz.readthedocs.org/en/latest/curry.html """ - property __doc__: - def __get__(self): - return self.func.__doc__ - - property __name__: - def __get__(self): - return self.func.__name__ - def __cinit__(self, func, *args, **kwargs): if not PyCallable_Check(func): raise TypeError("Input must be callable") @@ -193,6 +186,8 @@ cdef class curry: self.func = func self.args = args self.keywords = kwargs if kwargs else None + self.__doc__ = getattr(func, '__doc__', None) + self.__name__ = getattr(func, '__name__', '') def __str__(self): return str(self.func) @@ -200,6 +195,20 @@ cdef class curry: def __repr__(self): return repr(self.func) + def __hash__(self): + return hash((self.func, self.args, + frozenset(self.keywords.items()) if self.keywords + else None)) + + def __richcmp__(self, other, int op): + is_equal = (isinstance(other, curry) and self.func == other.func and + self.args == other.args and self.keywords == other.keywords) + if op == 2: # == + return is_equal + if op == 3: # != + return not is_equal + return PyObject_RichCompare(id(self), id(other), op) + def __call__(self, *args, **kwargs): cdef PyObject *obj cdef object val @@ -436,7 +445,7 @@ cdef class complement: cdef class _juxt_inner: def __cinit__(self, funcs): - self.funcs = funcs + self.funcs = tuple(funcs) def __call__(self, *args, **kwargs): if kwargs: @@ -470,11 +479,10 @@ def juxt(*funcs): [11, 20] """ if len(funcs) == 1 and not PyCallable_Check(funcs[0]): - funcs = tuple(funcs[0]) + funcs = funcs[0] return c_juxt(funcs) - cpdef object do(object func, object x): """ Runs ``func`` on ``x``, returns ``x`` diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index 910b7ea..1cf6a17 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -97,7 +97,7 @@ cdef class interpose: cpdef dict frequencies(object seq) -cpdef dict reduceby(object key, object binop, object seq, object init) +cpdef dict reduceby(object key, object binop, object seq, object init=*) cdef class iterate: diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 22fa088..df6c613 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -96,6 +96,7 @@ cdef class accumulate: return self.result + cpdef dict groupby(object func, object seq): """ Group a collection by a key function @@ -710,7 +711,7 @@ cpdef dict frequencies(object seq): ''' -cpdef dict reduceby(object key, object binop, object seq, object init): +cpdef dict reduceby(object key, object binop, object seq, object init=no_default): """ Perform a simultaneous groupby and reduction @@ -754,7 +755,10 @@ cpdef dict reduceby(object key, object binop, object seq, object init): k = key(item) obj = PyDict_GetItem(d, k) if obj is NULL: - val = binop(init, item) + if init is no_default: + val = item + else: + val = binop(init, item) else: val = obj val = binop(val, item) diff --git a/cytoolz/tests/test_curried.py b/cytoolz/tests/test_curried.py index 449e88a..4506ce9 100644 --- a/cytoolz/tests/test_curried.py +++ b/cytoolz/tests/test_curried.py @@ -16,7 +16,6 @@ def test_merge_with(): assert merge_with(sum)({1: 1}, {1: 2}) == {1: 3} -# XXX: backport to toolz def test_merge_with_list(): assert merge_with(sum, [{'a': 1}, {'a': 2}]) == {'a': 3} diff --git a/cytoolz/tests/test_dicttoolz.py b/cytoolz/tests/test_dicttoolz.py index 9133fe7..f4c36db 100644 --- a/cytoolz/tests/test_dicttoolz.py +++ b/cytoolz/tests/test_dicttoolz.py @@ -1,6 +1,6 @@ from cytoolz.utils import raises from cytoolz.dicttoolz import (merge, merge_with, valmap, keymap, update_in, - assoc, keyfilter, valfilter) + assoc, keyfilter, valfilter) inc = lambda x: x + 1 @@ -85,4 +85,3 @@ def test_update_in(): oldd = d update_in(d, ['x'], inc) assert d is oldd - diff --git a/cytoolz/tests/test_functoolz.py b/cytoolz/tests/test_functoolz.py index 476eb9c..7868cd3 100644 --- a/cytoolz/tests/test_functoolz.py +++ b/cytoolz/tests/test_functoolz.py @@ -1,6 +1,6 @@ from cytoolz.functoolz import (thread_first, thread_last, memoize, curry, - compose, pipe, complement, do, juxt, - _num_required_args) + compose, pipe, complement, do, juxt) +from cytoolz.functoolz import _num_required_args from operator import add, mul, itemgetter from cytoolz.utils import raises from functools import partial @@ -78,6 +78,17 @@ def f(x, y=0): assert fm2(3) == f2(3) +def test_memoize_partial(): + def f(x, y=0): + return x + y + + f2 = partial(f, y=1) + fm2 = memoize(f2) + + assert fm2(3) == f2(3) + assert fm2(3) == f2(3) + + def test_memoize_key_signature(): # Single argument should not be tupled as a key. No keywords. mf = memoize(lambda x: False, cache={1: True}) @@ -221,6 +232,87 @@ def foo(a, b, c=1): assert p(1, 2) == c(1, 2) +def test_curry_is_idempotent(): + def foo(a, b, c=1): + return a + b + c + + f = curry(foo, 1, c=2) + g = curry(f) + assert isinstance(f, curry) + assert isinstance(g, curry) + assert not isinstance(g.func, curry) + assert not hasattr(g.func, 'func') + assert f.func == g.func + assert f.args == g.args + assert f.keywords == g.keywords + + +def test_curry_attributes_readonly(): + def foo(a, b, c=1): + return a + b + c + + f = curry(foo, 1, c=2) + assert raises(AttributeError, lambda: setattr(f, 'args', (2,))) + assert raises(AttributeError, lambda: setattr(f, 'keywords', {'c': 3})) + assert raises(AttributeError, lambda: setattr(f, 'func', f)) + + +def test_curry_attributes_writable(): + def foo(a, b, c=1): + return a + b + c + + f = curry(foo, 1, c=2) + f.__name__ = 'newname' + f.__doc__ = 'newdoc' + assert f.__name__ == 'newname' + assert f.__doc__ == 'newdoc' + if hasattr(f, 'func_name'): + assert f.__name__ == f.func_name + + +def test_curry_comparable(): + def foo(a, b, c=1): + return a + b + c + f1 = curry(foo, 1, c=2) + f2 = curry(foo, 1, c=2) + g1 = curry(foo, 1, c=3) + h1 = curry(foo, c=2) + h2 = h1(c=2) + h3 = h1() + assert f1 == f2 + assert not (f1 != f2) + assert f1 != g1 + assert not (f1 == g1) + assert f1 != h1 + assert h1 == h2 + assert h1 == h3 + + # test function comparison works + def bar(a, b, c=1): + return a + b + c + b1 = curry(bar, 1, c=2) + assert b1 != f1 + + assert set([f1, f2, g1, h1, h2, h3, b1, b1()]) == set([f1, g1, h1, b1]) + + # test unhashable input + unhash1 = curry(foo, []) + assert raises(TypeError, lambda: hash(unhash1)) + unhash2 = curry(foo, c=[]) + assert raises(TypeError, lambda: hash(unhash2)) + + +def test_curry_doesnot_transmogrify(): + # Early versions of `curry` transmogrified to `partial` objects if + # only one positional argument remained even if keyword arguments + # were present. Now, `curry` should always remain `curry`. + def f(x, y=0): + return x + y + + cf = curry(f) + assert cf(y=1)(y=2)(y=3)(1) == f(1, 3) + + def test__num_required_args(): assert _num_required_args(map) is None assert _num_required_args(lambda x: x) == 1 @@ -289,5 +381,5 @@ def test_do(): def test_juxt_generator_input(): data = list(range(10)) juxtfunc = juxt(itemgetter(2*i) for i in range(5)) - assert tuple(juxtfunc(data)) == (0, 2, 4, 6, 8) - assert tuple(juxtfunc(data)) == (0, 2, 4, 6, 8) + assert juxtfunc(data) == (0, 2, 4, 6, 8) + assert juxtfunc(data) == (0, 2, 4, 6, 8) diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index a3404ff..788c69b 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -1,20 +1,24 @@ import itertools +from itertools import starmap from cytoolz.utils import raises from functools import partial from cytoolz.itertoolz import (remove, groupby, merge_sorted, - concat, concatv, interleave, unique, - identity, isiterable, - mapcat, isdistinct, first, second, - nth, take, drop, interpose, get, - rest, last, cons, frequencies, - reduceby, iterate, accumulate, - sliding_window, count, partition, - partition_all, take_nth, pluck) - + concat, concatv, interleave, unique, + isiterable, + mapcat, isdistinct, first, second, + nth, take, drop, interpose, get, + rest, last, cons, frequencies, + reduceby, iterate, accumulate, + sliding_window, count, partition, + partition_all, take_nth, pluck) from cytoolz.compatibility import range, filter from operator import add, mul +def identity(x): + return x + + def iseven(x): return x % 2 == 0 @@ -55,6 +59,7 @@ def test_merge_sorted(): assert ''.join(merge_sorted('abc', 'abc', 'abc', key=ord)) == 'aaabbbccc' assert ''.join(merge_sorted('cba', 'cba', 'cba', key=lambda x: -ord(x))) == 'cccbbbaaa' + assert list(merge_sorted([1], [2, 3, 4], key=identity)) == [1, 2, 3, 4] def test_interleave(): @@ -140,9 +145,12 @@ def test_get(): assert get({}, [1, 2, 3], default='bar') == 'bar' assert get([0, 2], 'AB', 'C') == ('A', 'C') + assert get([0], 'AB') == ('A',) + assert raises(IndexError, lambda: get(10, 'ABC')) assert raises(KeyError, lambda: get(10, {'a': 1})) assert raises(TypeError, lambda: get({}, [1, 2, 3])) + assert raises(TypeError, lambda: get([1, 2, 3], 1, None)) def test_mapcat(): @@ -204,6 +212,10 @@ def test_reduceby(): projects, 0) == {'CA': 1200000, 'IL': 2100000} +def test_reduce_by_init(): + assert reduceby(iseven, add, [1, 2, 3, 4]) == {True: 2 + 4, False: 1 + 3} + + def test_iterate(): assert list(itertools.islice(iterate(inc, 0), 0, 5)) == [0, 1, 2, 3, 4] assert list(take(4, iterate(double, 1))) == [1, 2, 4, 8] @@ -259,6 +271,7 @@ def test_pluck(): assert list(pluck('id', data)) == [1, 2] assert list(pluck('price', data, None)) == [None, 1] assert list(pluck(['id', 'name'], data)) == [(1, 'cheese'), (2, 'pies')] + assert list(pluck(['name'], data)) == [('cheese',), ('pies',)] assert list(pluck(['price', 'other'], data, None)) == [(None, None), (1, None)] From 561fd1eaff8efb275feb54b1da8b8da2c177119b Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 24 Jun 2014 12:00:42 -0400 Subject: [PATCH 019/146] Use op-code constants `Py_EQ` and `Py_NE` for rich comparisons --- cytoolz/functoolz.pyx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index d17fe78..5664f02 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -2,8 +2,8 @@ import inspect from cpython.dict cimport PyDict_Merge, PyDict_New from cpython.exc cimport PyErr_Clear, PyErr_ExceptionMatches, PyErr_Occurred -from cpython.object cimport (PyCallable_Check, PyObject_Call, - PyObject_CallObject, PyObject_RichCompare) +from cpython.object cimport (PyCallable_Check, PyObject_Call, PyObject_CallObject, + PyObject_RichCompare, Py_EQ, Py_NE) from cpython.ref cimport PyObject from cpython.sequence cimport PySequence_Concat from cpython.set cimport PyFrozenSet_New @@ -203,9 +203,9 @@ cdef class curry: def __richcmp__(self, other, int op): is_equal = (isinstance(other, curry) and self.func == other.func and self.args == other.args and self.keywords == other.keywords) - if op == 2: # == + if op == Py_EQ: return is_equal - if op == 3: # != + if op == Py_NE: return not is_equal return PyObject_RichCompare(id(self), id(other), op) From 915aeb21ba8a901c75d59aa9d0c6c487f6f312f7 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 24 Jun 2014 14:17:55 -0400 Subject: [PATCH 020/146] Add `cytoolz.utils.consume` to efficiently consume iterators. It was moved from `cytoolz.itertoolz._consume`. `toolz` does not have `consume`. --- cytoolz/itertoolz.pyx | 11 ----------- cytoolz/tests/test_utils.py | 11 ++++++++++- cytoolz/utils.pxd | 1 + cytoolz/{utils.py => utils.pyx} | 15 +++++++++++++-- setup.py | 2 +- 5 files changed, 25 insertions(+), 15 deletions(-) create mode 100644 cytoolz/utils.pxd rename cytoolz/{utils.py => utils.pyx} (91%) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index df6c613..0c4c4cf 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -96,7 +96,6 @@ cdef class accumulate: return self.result - cpdef dict groupby(object func, object seq): """ Group a collection by a key function @@ -1063,13 +1062,3 @@ cpdef object pluck(object ind, object seqs, object default=no_default): if default is no_default: return _pluck_index(ind, seqs) return _pluck_index_default(ind, seqs, default) - - -# I find `_consume` convenient for benchmarking. Perhaps this belongs -# elsewhere, so it is private (leading underscore) and hidden away for now. - -cpdef object _consume(object seq): - """ - Efficiently consume an iterable """ - for _ in seq: - pass diff --git a/cytoolz/tests/test_utils.py b/cytoolz/tests/test_utils.py index a1e76db..0fabf4e 100644 --- a/cytoolz/tests/test_utils.py +++ b/cytoolz/tests/test_utils.py @@ -1,6 +1,15 @@ -from cytoolz.utils import raises +from cytoolz.utils import raises, consume def test_raises(): assert raises(ZeroDivisionError, lambda: 1 / 0) assert not raises(ZeroDivisionError, lambda: 1) + + +def test_consume(): + l = [1, 2, 3] + assert consume(l) is None + il = iter(l) + assert consume(il) is None + assert raises(StopIteration, lambda: next(il)) + assert raises(TypeError, lambda: consume(1)) diff --git a/cytoolz/utils.pxd b/cytoolz/utils.pxd new file mode 100644 index 0000000..412e8fa --- /dev/null +++ b/cytoolz/utils.pxd @@ -0,0 +1 @@ +cpdef object consume(object seq) diff --git a/cytoolz/utils.py b/cytoolz/utils.pyx similarity index 91% rename from cytoolz/utils.py rename to cytoolz/utils.pyx index 2d1f6e7..0d0ba31 100644 --- a/cytoolz/utils.py +++ b/cytoolz/utils.pyx @@ -1,9 +1,13 @@ +#cython: embedsignature=True import doctest import inspect import os.path import cytoolz +__all__ = ['raises', 'no_default', 'include_dirs', 'consume', 'module_doctest'] + + def raises(err, lamda): try: lamda() @@ -45,6 +49,13 @@ def include_dirs(): return os.path.split(cytoolz.__path__[0]) +cpdef object consume(object seq): + """ + Efficiently consume an iterable """ + for _ in seq: + pass + + # The utilities below were obtained from: # https://github.com/cython/cython/wiki/FAQ # #how-can-i-run-doctests-in-cython-code-pyx-files @@ -81,7 +92,7 @@ def _from_module(module, object): raise ValueError("object must be a class or function") -def fix_module_doctest(module): +def _fix_module_doctest(module): """ Extract docstrings from cython functions, that would be skipped by doctest otherwise. @@ -102,5 +113,5 @@ def module_doctest(m, *args, **kwargs): Return True on success, False on failure. """ - fix_module_doctest(m) + _fix_module_doctest(m) return doctest.testmod(m, *args, **kwargs).failed == 0 diff --git a/setup.py b/setup.py index c3b6db1..5f4e9c1 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ ext_modules = [] for modname in ['dicttoolz', 'functoolz', 'itertoolz', - 'curried_exceptions', 'recipes']: + 'curried_exceptions', 'recipes', 'utils']: ext_modules.append(Extension('cytoolz.' + modname, ['cytoolz/' + modname + suffix])) From a234e7dcffc196ceba2001debedf4aa41a531461 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 28 Jun 2014 18:43:05 -0400 Subject: [PATCH 021/146] groupby can accept an index or list of indices as the key --- cytoolz/itertoolz.pxd | 2 +- cytoolz/itertoolz.pyx | 55 +++++++++++++++++++++++++-------- cytoolz/tests/test_itertoolz.py | 21 ++++++++----- 3 files changed, 57 insertions(+), 21 deletions(-) diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index 87644f0..d9bbbba 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -9,7 +9,7 @@ cdef class accumulate: cdef object result -cpdef dict groupby(object func, object seq) +cpdef dict groupby(object key, object seq) cdef class _merge_sorted: diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index dab5913..1478266 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -1,5 +1,5 @@ #cython: embedsignature=True -from cpython.dict cimport PyDict_GetItem +from cpython.dict cimport PyDict_GetItem, PyDict_SetItem from cpython.exc cimport (PyErr_Clear, PyErr_ExceptionMatches, PyErr_GivenExceptionMatches, PyErr_Occurred) from cpython.list cimport (PyList_Append, PyList_GET_ITEM, PyList_GET_SIZE) @@ -96,7 +96,17 @@ cdef class accumulate: return self.result -cpdef dict groupby(object func, object seq): +cdef inline void _groupby_core(dict d, object key, object item): + cdef PyObject *obj = PyDict_GetItem(d, key) + if obj is NULL: + val = [] + PyList_Append(val, item) + PyDict_SetItem(d, key, val) + else: + PyList_Append(obj, item) + + +cpdef dict groupby(object key, object seq): """ Group a collection by a key function @@ -108,20 +118,39 @@ cpdef dict groupby(object func, object seq): >>> groupby(iseven, [1, 2, 3, 4, 5, 6, 7, 8]) {False: [1, 3, 5, 7], True: [2, 4, 6, 8]} + Non-callable keys imply grouping on a member. + + >>> groupby('gender', [{'name': 'Alice', 'gender': 'F'}, + ... {'name': 'Bob', 'gender': 'M'}, + ... {'name': 'Charlie', 'gender': 'M'}]) # doctest:+SKIP + {'F': [{'gender': 'F', 'name': 'Alice'}], + 'M': [{'gender': 'M', 'name': 'Bob'}, + {'gender': 'M', 'name': 'Charlie'}]} + See Also: - ``countby`` + countby """ cdef dict d = {} - cdef list vals - cdef PyObject *obj - cdef object item, key - for item in seq: - key = func(item) - obj = PyDict_GetItem(d, key) - if obj is NULL: - d[key] = [item] - else: - PyList_Append(obj, item) + cdef object item, keyval + cdef Py_ssize_t i, N + if callable(key): + for item in seq: + keyval = key(item) + _groupby_core(d, keyval, item) + elif isinstance(key, list): + N = PyList_GET_SIZE(key) + for item in seq: + keyval = PyTuple_New(N) + for i in range(N): + val = PyList_GET_ITEM(key, i) + val = item[val] + Py_INCREF(val) + PyTuple_SET_ITEM(keyval, i, val) + _groupby_core(d, keyval, item) + else: + for item in seq: + keyval = item[key] + _groupby_core(d, keyval, item) return d diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index 681b4d6..c4bf2ed 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -45,6 +45,20 @@ def test_groupby(): assert groupby(iseven, [1, 2, 3, 4]) == {True: [2, 4], False: [1, 3]} +def test_groupby_non_callable(): + assert groupby(0, [(1, 2), (1, 3), (2, 2), (2, 4)]) == \ + {1: [(1, 2), (1, 3)], + 2: [(2, 2), (2, 4)]} + + assert groupby([0], [(1, 2), (1, 3), (2, 2), (2, 4)]) == \ + {(1,): [(1, 2), (1, 3)], + (2,): [(2, 2), (2, 4)]} + + assert groupby([0, 0], [(1, 2), (1, 3), (2, 2), (2, 4)]) == \ + {(1, 1): [(1, 2), (1, 3)], + (2, 2): [(2, 2), (2, 4)]} + + def test_merge_sorted(): assert list(merge_sorted([1, 2, 3], [1, 2, 3])) == [1, 1, 2, 2, 3, 3] assert list(merge_sorted([1, 3, 5], [2, 4, 6])) == [1, 2, 3, 4, 5, 6] @@ -293,8 +307,6 @@ def addpair(pair): ((2, 'two', 'banana', 2)), ((2, 'two', 'coconut', 2))]) - print(result) - print(expected) assert result == expected @@ -329,8 +341,6 @@ def test_join_double_repeats(): ((2, 'dos', 'banana', 2)), ((2, 'dos', 'coconut', 2))]) - print(result) - print(expected) assert result == expected @@ -339,7 +349,6 @@ def test_join_missing_element(): fruit = [('apple', 5), ('orange', 1)] result = list(join(first, names, second, fruit)) - print(result) result = set(starmap(add, result)) expected = set([((1, 'one', 'orange', 1))]) @@ -351,8 +360,6 @@ def test_left_outer_join(): result = set(join(identity, [1, 2], identity, [2, 3], left_default=None)) expected = set([(2, 2), (None, 3)]) - print(result) - print(expected) assert result == expected From 66f1f4c6d97fe9080d970310584b89e1b116c5ac Mon Sep 17 00:00:00 2001 From: Matthew Rocklin Date: Sat, 28 Jun 2014 18:07:40 -0700 Subject: [PATCH 022/146] add conda.yaml and .binstar.yml files --- .binstar.yml | 14 ++++++++++++++ conda.yaml | 23 +++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 .binstar.yml create mode 100644 conda.yaml diff --git a/.binstar.yml b/.binstar.yml new file mode 100644 index 0000000..e371a15 --- /dev/null +++ b/.binstar.yml @@ -0,0 +1,14 @@ +package: cytoolz +platform: + - linux-64 + - linux-32 + - osx-64 +engine: + - python=2 + - python=3 +script: + - touch build.sh + - conda build . +build_targets: + files: conda + channels: main diff --git a/conda.yaml b/conda.yaml new file mode 100644 index 0000000..cd4c343 --- /dev/null +++ b/conda.yaml @@ -0,0 +1,23 @@ +package: + name: cytoolz + version: "0.6.1" + +build: + number: {{environ.get('BINSTAR_BUILD', 1)}} + script: + - cd $RECIPE_DIR + - $PYTHON setup.py install --with-cython + +requirements: + build: + - setuptools + - python + - cython + - toolz + + run: + - python + +about: + home: http://toolz.readthedocs.org/ + license: BSD From 899878a6607b4a9e35dc28862f42d8f7130f7b4b Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 30 Jun 2014 11:06:05 -0400 Subject: [PATCH 023/146] reduceby and countby can accept an index or list of indices as the key Lesson learned: even inline functions need to return `object` so exceptions can be properly raised. --- cytoolz/itertoolz.pyx | 66 +++++++++++++++++++++++---------- cytoolz/recipes.pxd | 2 +- cytoolz/recipes.pyx | 8 ++-- cytoolz/tests/test_itertoolz.py | 15 +++++++- cytoolz/tests/test_recipes.py | 3 ++ 5 files changed, 69 insertions(+), 25 deletions(-) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 1478266..00b9a7b 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -96,7 +96,7 @@ cdef class accumulate: return self.result -cdef inline void _groupby_core(dict d, object key, object item): +cdef inline object _groupby_core(dict d, object key, object item): cdef PyObject *obj = PyDict_GetItem(d, key) if obj is NULL: val = [] @@ -739,6 +739,17 @@ cpdef dict frequencies(object seq): ''' +cdef inline object _reduceby_core(dict d, object key, object item, object binop, + object init, bint skip_init): + cdef PyObject *obj = PyDict_GetItem(d, key) + if obj is not NULL: + PyDict_SetItem(d, key, binop(obj, item)) + elif skip_init: + PyDict_SetItem(d, key, item) + else: + PyDict_SetItem(d, key, binop(init, item)) + + cpdef dict reduceby(object key, object binop, object seq, object init=no_default): """ Perform a simultaneous groupby and reduction @@ -759,38 +770,55 @@ cpdef dict reduceby(object key, object binop, object seq, object init=no_default operate in much less space. This makes it suitable for larger datasets that do not fit comfortably in memory + Simple Examples + --------------- + >>> from operator import add, mul - >>> data = [1, 2, 3, 4, 5] >>> iseven = lambda x: x % 2 == 0 - >>> reduceby(iseven, add, data, 0) + + >>> data = [1, 2, 3, 4, 5] + + >>> reduceby(iseven, add, data) {False: 9, True: 6} - >>> reduceby(iseven, mul, data, 1) + + >>> reduceby(iseven, mul, data) {False: 15, True: 8} + Complex Example + --------------- + >>> projects = [{'name': 'build roads', 'state': 'CA', 'cost': 1000000}, ... {'name': 'fight crime', 'state': 'IL', 'cost': 100000}, ... {'name': 'help farmers', 'state': 'IL', 'cost': 2000000}, ... {'name': 'help farmers', 'state': 'CA', 'cost': 200000}] - >>> reduceby(lambda x: x['state'], # doctest: +SKIP + + >>> reduceby('state', # doctest: +SKIP ... lambda acc, x: acc + x['cost'], ... projects, 0) {'CA': 1200000, 'IL': 2100000} """ cdef dict d = {} - cdef object item, k, val - cdef PyObject *obj - for item in seq: - k = key(item) - obj = PyDict_GetItem(d, k) - if obj is NULL: - if init is no_default: - val = item - else: - val = binop(init, item) - else: - val = obj - val = binop(val, item) - d[k] = val + cdef object item, keyval + cdef Py_ssize_t i, N + cdef bint skip_init = init is no_default + if callable(key): + for item in seq: + keyval = key(item) + _reduceby_core(d, keyval, item, binop, init, skip_init) + elif isinstance(key, list): + N = PyList_GET_SIZE(key) + for item in seq: + keyval = PyTuple_New(N) + for i in range(N): + val = PyList_GET_ITEM(key, i) + val = item[val] + Py_INCREF(val) + PyTuple_SET_ITEM(keyval, i, val) + _reduceby_core(d, keyval, item, binop, init, skip_init) + else: + for item in seq: + keyval = item[key] + _reduceby_core(d, keyval, item, binop, init, skip_init) return d diff --git a/cytoolz/recipes.pxd b/cytoolz/recipes.pxd index ebca356..aa0bc39 100644 --- a/cytoolz/recipes.pxd +++ b/cytoolz/recipes.pxd @@ -1,4 +1,4 @@ -cpdef object countby(object func, object seq) +cpdef object countby(object key, object seq) cdef class partitionby: diff --git a/cytoolz/recipes.pyx b/cytoolz/recipes.pyx index 816294b..d0e06af 100644 --- a/cytoolz/recipes.pyx +++ b/cytoolz/recipes.pyx @@ -1,6 +1,6 @@ #cython: embedsignature=True from cpython.sequence cimport PySequence_Tuple -from .itertoolz cimport frequencies +from .itertoolz cimport frequencies, pluck from itertools import groupby from .compatibility import map @@ -9,7 +9,7 @@ from .compatibility import map __all__ = ['countby', 'partitionby'] -cpdef object countby(object func, object seq): +cpdef object countby(object key, object seq): """ Count elements of a collection by a key function @@ -23,7 +23,9 @@ cpdef object countby(object func, object seq): See Also: groupby """ - return frequencies(map(func, seq)) + if not callable(key): + return frequencies(pluck(key, seq)) + return frequencies(map(key, seq)) cdef class partitionby: diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index c4bf2ed..5420dcd 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -225,6 +225,18 @@ def test_reduceby(): lambda acc, x: acc + x['cost'], projects, 0) == {'CA': 1200000, 'IL': 2100000} + assert reduceby('state', + lambda acc, x: acc + x['cost'], + projects, 0) == {'CA': 1200000, 'IL': 2100000} + + assert reduceby(['state'], + lambda acc, x: acc + x['cost'], + projects, 0) == {('CA',): 1200000, ('IL',): 2100000} + + assert reduceby(['state', 'state'], + lambda acc, x: acc + x['cost'], + projects, 0) == {('CA', 'CA'): 1200000, ('IL', 'IL'): 2100000} + def test_reduce_by_init(): assert reduceby(iseven, add, [1, 2, 3, 4]) == {True: 2 + 4, False: 1 + 3} @@ -348,8 +360,7 @@ def test_join_missing_element(): names = [(1, 'one'), (2, 'two'), (3, 'three')] fruit = [('apple', 5), ('orange', 1)] - result = list(join(first, names, second, fruit)) - result = set(starmap(add, result)) + result = set(starmap(add, join(first, names, second, fruit))) expected = set([((1, 'one', 'orange', 1))]) diff --git a/cytoolz/tests/test_recipes.py b/cytoolz/tests/test_recipes.py index a0b58fc..4ec0a9b 100644 --- a/cytoolz/tests/test_recipes.py +++ b/cytoolz/tests/test_recipes.py @@ -8,6 +8,9 @@ def iseven(x): def test_countby(): assert countby(iseven, [1, 2, 3]) == {True: 1, False: 2} assert countby(len, ['cat', 'dog', 'mouse']) == {3: 2, 5: 1} + assert countby(0, ('ab', 'ac', 'bc')) == {'a': 2, 'b': 1} + assert countby([0], ('ab', 'ac', 'bc')) == {('a',): 2, ('b',): 1} + assert countby([0, 0], ('ab', 'ac', 'bc')) == {('a', 'a'): 2, ('b', 'b'): 1} def test_partitionby(): From 4df4d3ab2bbf11e550e8e4a48702d58985a04b47 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 30 Jun 2014 13:41:12 -0400 Subject: [PATCH 024/146] Fixes to pass release tests. cytoolz now matches toolz. --- cytoolz/curried.py | 1 + cytoolz/functoolz.pyx | 10 +++++----- cytoolz/itertoolz.pyx | 32 +++++--------------------------- cytoolz/tests/test_none_safe.py | 4 ++++ 4 files changed, 15 insertions(+), 32 deletions(-) diff --git a/cytoolz/curried.py b/cytoolz/curried.py index 69d9a1a..098ae68 100644 --- a/cytoolz/curried.py +++ b/cytoolz/curried.py @@ -52,6 +52,7 @@ interleave = cytoolz.curry(interleave) interpose = cytoolz.curry(interpose) iterate = cytoolz.curry(iterate) +join = cytoolz.curry(join) keyfilter = cytoolz.curry(keyfilter) keymap = cytoolz.curry(keymap) map = cytoolz.curry(map) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 5664f02..a8853cf 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -466,17 +466,17 @@ def juxt(*funcs): Creates a function that calls several functions with the same arguments. Takes several functions and returns a function that applies its arguments - to each of those functions then returns a sequence of the results. + to each of those functions then returns a tuple of the results. Name comes from juxtaposition: the fact of two things being seen or placed close together with contrasting effect. >>> inc = lambda x: x + 1 >>> double = lambda x: x * 2 - >>> list(juxt(inc, double)(10)) - [11, 20] - >>> list(juxt([inc, double])(10)) - [11, 20] + >>> juxt(inc, double)(10) + (11, 20) + >>> juxt([inc, double])(10) + (11, 20) """ if len(funcs) == 1 and not PyCallable_Check(funcs[0]): funcs = funcs[0] diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 00b9a7b..653cc68 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -21,7 +21,8 @@ __all__ = ['remove', 'accumulate', 'groupby', 'merge_sorted', 'interleave', 'unique', 'isiterable', 'isdistinct', 'take', 'drop', 'take_nth', 'first', 'second', 'nth', 'last', 'get', 'concat', 'concatv', 'mapcat', 'cons', 'interpose', 'frequencies', 'reduceby', 'iterate', - 'sliding_window', 'partition', 'partition_all', 'count', 'pluck'] + 'sliding_window', 'partition', 'partition_all', 'count', 'pluck', + 'join'] concatv = chain @@ -35,7 +36,7 @@ cpdef object identity(object x): cdef class remove: """ remove(predicate, seq) - Return those items of collection for which predicate(item) is true. + Return those items of sequence for which predicate(item) is False >>> def iseven(x): ... return x % 2 == 0 @@ -725,20 +726,6 @@ cpdef dict frequencies(object seq): return d -''' Alternative implementation of `frequencies` -cpdef dict frequencies(object seq): - cdef dict d = {} - cdef Py_ssize_t val - for item in seq: - if item in d: - val = d[item] - d[item] = val + 1 - else: - d[item] = 1 - return d -''' - - cdef inline object _reduceby_core(dict d, object key, object item, object binop, object init, bint skip_init): cdef PyObject *obj = PyDict_GetItem(d, key) @@ -1135,7 +1122,8 @@ cpdef object join(object leftkey, object leftseq, object rightkey, object rightseq, object left_default=no_default, object right_default=no_default): - """ Join two sequences on common attributes + """ + Join two sequences on common attributes This is a semi-streaming operation. The LEFT sequence is fully evaluated and placed into memory. The RIGHT sequence is evaluated lazily and so can @@ -1370,13 +1358,3 @@ cdef class _inner_join(_join): match = PyList_GET_ITEM(self.matches, self.i) # skip error checking self.i += 1 return (match, self.right) - - -# I find `_consume` convenient for benchmarking. Perhaps this belongs -# elsewhere, so it is private (leading underscore) and hidden away for now. - -cpdef object _consume(object seq): - """ - Efficiently consume an iterable """ - for _ in seq: - pass diff --git a/cytoolz/tests/test_none_safe.py b/cytoolz/tests/test_none_safe.py index b65175d..b129447 100644 --- a/cytoolz/tests/test_none_safe.py +++ b/cytoolz/tests/test_none_safe.py @@ -256,6 +256,10 @@ def test_itertoolz(): assert raises(TypeError, lambda: list(unique([1, 1, 2], key=None))) tested.append('unique') + assert raises(TypeError, lambda: join(first, None, second, (1, 2, 3))) + assert raises(TypeError, lambda: join(first, (1, 2, 3), second, None)) + tested.append('join') + s1 = set(tested) s2 = set(cytoolz.itertoolz.__all__) assert s1 == s2, '%s not tested for being None-safe' % ', '.join(s2 - s1) From 63b41cbb00d952ce7df071219d739ededafc197c Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 2 Jul 2014 11:17:52 -0400 Subject: [PATCH 025/146] Faster join by using subclasses for each case (type of join and index keys) This is not the cleanest way to handle multiple key types (callable, list of incidex, and a single index), and I don't want the current approach to set a precedence that will be used by other classes. A cleaner approach would be to have a single `make_key` function with the following signature: ```python cdef inline object make_key(object rightkey, object item, bint iscallable, bint islist) ``` I doubt that using `make_key` would significantly impact the performance. --- cytoolz/itertoolz.pxd | 58 ++++++++--- cytoolz/itertoolz.pyx | 225 ++++++++++++++++++++++++++---------------- 2 files changed, 189 insertions(+), 94 deletions(-) diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index d9bbbba..d7c2829 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -155,23 +155,22 @@ cpdef object join(object leftkey, object leftseq, object right_default=*) cdef class _join: - cdef Py_ssize_t n - cdef object iterseq - cdef object leftkey + cdef dict d + cdef list matches + cdef set seen_keys cdef object leftseq - cdef object rightkey cdef object rightseq - cdef object matches + cdef object _rightkey cdef object right - cdef object key - cdef object d - cdef object d_items - cdef object seen_keys - cdef object is_rightseq_exhausted cdef object left_default cdef object right_default - cdef int i cdef object keys + cdef Py_ssize_t N + cdef Py_ssize_t i + cdef bint is_rightseq_exhausted + + cdef object rightkey(self) + cdef class _inner_join(_join): pass @@ -184,3 +183,40 @@ cdef class _left_outer_join(_join): cdef class _outer_join(_join): pass + + +cdef class _inner_join_key(_inner_join): + pass + +cdef class _inner_join_index(_inner_join): + pass + +cdef class _inner_join_indices(_inner_join): + pass + +cdef class _right_outer_join_key(_right_outer_join): + pass + +cdef class _right_outer_join_index(_right_outer_join): + pass + +cdef class _right_outer_join_indices(_right_outer_join): + pass + +cdef class _left_outer_join_key(_left_outer_join): + pass + +cdef class _left_outer_join_index(_left_outer_join): + pass + +cdef class _left_outer_join_indices(_left_outer_join): + pass + +cdef class _outer_join_key(_outer_join): + pass + +cdef class _outer_join_index(_outer_join): + pass + +cdef class _outer_join_indices(_outer_join): + pass diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 653cc68..b7f5ad5 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -1108,16 +1108,6 @@ cpdef object pluck(object ind, object seqs, object default=no_default): return _pluck_index_default(ind, seqs, default) -def getter(index): - if isinstance(index, list): - if len(index) == 1: - index = index[0] - return lambda x: (x[index],) - else: - return itemgetter(*index) - else: - return itemgetter(index) - cpdef object join(object leftkey, object leftseq, object rightkey, object rightseq, object left_default=no_default, @@ -1175,95 +1165,81 @@ cpdef object join(object leftkey, object leftseq, >>> # result = join(second, friends, first, cities) >>> result = join(1, friends, 0, cities) # doctest: +SKIP """ - return _join(leftkey, leftseq, rightkey, rightseq, - left_default, right_default) if left_default == no_default and right_default == no_default: - return _inner_join(leftkey, leftseq, rightkey, rightseq, - left_default, right_default) + if callable(rightkey): + return _inner_join_key(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) + elif isinstance(rightkey, list): + return _inner_join_indices(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) + else: + return _inner_join_index(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) elif left_default != no_default and right_default == no_default: - return _right_outer_join(leftkey, leftseq, rightkey, rightseq, - left_default, right_default) + if callable(rightkey): + return _right_outer_join_key(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) + elif isinstance(rightkey, list): + return _right_outer_join_indices(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) + else: + return _right_outer_join_index(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) elif left_default == no_default and right_default != no_default: - return _left_outer_join(leftkey, leftseq, rightkey, rightseq, - left_default, right_default) + if callable(rightkey): + return _left_outer_join_key(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) + elif isinstance(rightkey, list): + return _left_outer_join_indices(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) + else: + return _left_outer_join_index(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) else: - return _outer_join(leftkey, leftseq, rightkey, rightseq, - left_default, right_default) + if callable(rightkey): + return _outer_join_key(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) + elif isinstance(rightkey, list): + return _outer_join_indices(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) + else: + return _outer_join_index(leftkey, leftseq, rightkey, rightseq, + left_default, right_default) cdef class _join: - def __init__(self, - object leftkey, object leftseq, - object rightkey, object rightseq, - object left_default=no_default, - object right_default=no_default): - if not callable(leftkey): - leftkey = getter(leftkey) - if not callable(rightkey): - rightkey = getter(rightkey) - + def __cinit__(self, + object leftkey, object leftseq, + object rightkey, object rightseq, + object left_default=no_default, + object right_default=no_default): self.left_default = left_default self.right_default = right_default - self.leftkey = leftkey - self.rightkey = rightkey + self._rightkey = rightkey self.rightseq = iter(rightseq) + if isinstance(rightkey, list): + self.N = len(rightkey) self.d = groupby(leftkey, leftseq) self.seen_keys = set() - self.matches = () + self.matches = [] self.right = None self.is_rightseq_exhausted = False - def __iter__(self): return self - def __next__(self): - cdef PyObject *obj - if not self.is_rightseq_exhausted: - if self.i == len(self.matches): - try: - self.right = next(self.rightseq) - except StopIteration: - if self.right_default is no_default: - raise - self.is_rightseq_exhausted = True - self.keys = iter(self.d) - return next(self) - key = self.rightkey(self.right) - self.seen_keys.add(key) - obj = PyDict_GetItem(self.d, key) - if obj is NULL: - if self.left_default is not no_default: - return (self.left_default, self.right) - else: - return next(self) - self.matches = obj - self.i = 0 - match = PyList_GET_ITEM(self.matches, self.i) # skip error checking - self.i += 1 - return (match, self.right) - - elif self.right_default is not no_default: - if self.i == len(self.matches): - key = next(self.keys) - while key in self.seen_keys: - key = next(self.keys) - obj = PyDict_GetItem(self.d, key) - self.matches = obj - self.i = 0 - match = PyList_GET_ITEM(self.matches, self.i) # skip error checking - self.i += 1 - return (match, self.right_default) + cdef object rightkey(self): + pass cdef class _right_outer_join(_join): def __next__(self): cdef PyObject *obj - if self.i == len(self.matches): + if self.i == PyList_GET_SIZE(self.matches): self.right = next(self.rightseq) - key = self.rightkey(self.right) + key = self.rightkey() obj = PyDict_GetItem(self.d, key) if obj is NULL: return (self.left_default, self.right) @@ -1274,19 +1250,40 @@ cdef class _right_outer_join(_join): return (match, self.right) +cdef class _right_outer_join_key(_right_outer_join): + cdef object rightkey(self): + return self._rightkey(self.right) + + +cdef class _right_outer_join_index(_right_outer_join): + cdef object rightkey(self): + return self.right[self._rightkey] + + +cdef class _right_outer_join_indices(_right_outer_join): + cdef object rightkey(self): + keyval = PyTuple_New(self.N) + for i in range(self.N): + val = PyList_GET_ITEM(self._rightkey, i) + val = self.right[val] + Py_INCREF(val) + PyTuple_SET_ITEM(keyval, i, val) + return keyval + + cdef class _outer_join(_join): def __next__(self): cdef PyObject *obj if not self.is_rightseq_exhausted: - if self.i == len(self.matches): + if self.i == PyList_GET_SIZE(self.matches): try: self.right = next(self.rightseq) except StopIteration: self.is_rightseq_exhausted = True self.keys = iter(self.d) return next(self) - key = self.rightkey(self.right) - self.seen_keys.add(key) + key = self.rightkey() + PySet_Add(self.seen_keys, key) obj = PyDict_GetItem(self.d, key) if obj is NULL: return (self.left_default, self.right) @@ -1297,7 +1294,7 @@ cdef class _outer_join(_join): return (match, self.right) else: - if self.i == len(self.matches): + if self.i == PyList_GET_SIZE(self.matches): key = next(self.keys) while key in self.seen_keys: key = next(self.keys) @@ -1309,12 +1306,32 @@ cdef class _outer_join(_join): return (match, self.right_default) +cdef class _outer_join_key(_outer_join): + cdef object rightkey(self): + return self._rightkey(self.right) + + +cdef class _outer_join_index(_outer_join): + cdef object rightkey(self): + return self.right[self._rightkey] + + +cdef class _outer_join_indices(_outer_join): + cdef object rightkey(self): + keyval = PyTuple_New(self.N) + for i in range(self.N): + val = PyList_GET_ITEM(self._rightkey, i) + val = self.right[val] + Py_INCREF(val) + PyTuple_SET_ITEM(keyval, i, val) + return keyval + cdef class _left_outer_join(_join): def __next__(self): cdef PyObject *obj if not self.is_rightseq_exhausted: - if self.i == len(self.matches): + if self.i == PyList_GET_SIZE(self.matches): obj = NULL while obj is NULL: try: @@ -1323,8 +1340,8 @@ cdef class _left_outer_join(_join): self.is_rightseq_exhausted = True self.keys = iter(self.d) return next(self) - key = self.rightkey(self.right) - self.seen_keys.add(key) + key = self.rightkey() + PySet_Add(self.seen_keys, key) obj = PyDict_GetItem(self.d, key) self.matches = obj self.i = 0 @@ -1333,7 +1350,7 @@ cdef class _left_outer_join(_join): return (match, self.right) else: - if self.i == len(self.matches): + if self.i == PyList_GET_SIZE(self.matches): key = next(self.keys) while key in self.seen_keys: key = next(self.keys) @@ -1345,16 +1362,58 @@ cdef class _left_outer_join(_join): return (match, self.right_default) +cdef class _left_outer_join_key(_left_outer_join): + cdef object rightkey(self): + return self._rightkey(self.right) + + +cdef class _left_outer_join_index(_left_outer_join): + cdef object rightkey(self): + return self.right[self._rightkey] + + +cdef class _left_outer_join_indices(_left_outer_join): + cdef object rightkey(self): + keyval = PyTuple_New(self.N) + for i in range(self.N): + val = PyList_GET_ITEM(self._rightkey, i) + val = self.right[val] + Py_INCREF(val) + PyTuple_SET_ITEM(keyval, i, val) + return keyval + + cdef class _inner_join(_join): def __next__(self): cdef PyObject *obj = NULL - if self.i == len(self.matches): + if self.i == PyList_GET_SIZE(self.matches): while obj is NULL: self.right = next(self.rightseq) - key = self.rightkey(self.right) + key = self.rightkey() obj = PyDict_GetItem(self.d, key) self.matches = obj self.i = 0 match = PyList_GET_ITEM(self.matches, self.i) # skip error checking self.i += 1 return (match, self.right) + + +cdef class _inner_join_key(_inner_join): + cdef object rightkey(self): + return self._rightkey(self.right) + + +cdef class _inner_join_index(_inner_join): + cdef object rightkey(self): + return self.right[self._rightkey] + + +cdef class _inner_join_indices(_inner_join): + cdef object rightkey(self): + keyval = PyTuple_New(self.N) + for i in range(self.N): + val = PyList_GET_ITEM(self._rightkey, i) + val = self.right[val] + Py_INCREF(val) + PyTuple_SET_ITEM(keyval, i, val) + return keyval From 59c5a8b3956347aef24ad4db86089a9db9259c80 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 2 Jul 2014 16:02:04 -0400 Subject: [PATCH 026/146] Attempt to add Python 3.4 support This copies what was done in pytoolz/toolz:157 --- .travis.yml | 1 + cytoolz/functoolz.pyx | 67 +++++++++++++++++++++++++++++---- cytoolz/tests/test_functoolz.py | 2 +- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66df268..1686736 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ python: - "2.7" - "3.2" - "3.3" + - "3.4" before_install: - pip install git+https://github.com/pytoolz/toolz.git diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index a8853cf..c44c30f 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -1,5 +1,8 @@ #cython: embedsignature=True import inspect +import sys +from .compatibility import filter as ifilter, map as imap, reduce + from cpython.dict cimport PyDict_Merge, PyDict_New from cpython.exc cimport PyErr_Clear, PyErr_ExceptionMatches, PyErr_Occurred from cpython.object cimport (PyCallable_Check, PyObject_Call, PyObject_CallObject, @@ -110,6 +113,13 @@ def thread_last(val, *forms): return c_thread_last(val, forms) +# This is a kludge for Python 3.4.0 support +# currently len(inspect.getargspec(map).args) == 0, a wrong result. +# As this is fixed in future versions then hopefully this kludge can be +# removed. +known_numargs = {map: 2, filter: 2, reduce: 2, imap: 2, ifilter: 2} + + cpdef object _num_required_args(object func): """ Number of args for func @@ -126,6 +136,8 @@ cpdef object _num_required_args(object func): >>> print(_num_required_args(bar)) None """ + if func in known_numargs: + return known_numargs[func] try: spec = inspect.getargspec(func) if spec.varargs: @@ -172,8 +184,10 @@ cdef class curry: raise TypeError("Input must be callable") # curry- or functools.partial-like object? Unpack and merge arguments - if (hasattr(func, 'func') and hasattr(func, 'args') - and hasattr(func, 'keywords')): + if (hasattr(func, 'func') + and hasattr(func, 'args') + and hasattr(func, 'keywords') + and isinstance(func.args, tuple)): if func.keywords: PyDict_Merge(kwargs, func.keywords, False) ## Equivalent to: @@ -241,6 +255,48 @@ cdef class curry: self.args, self.keywords = state +cpdef object has_kwargs(object f): + """ + Does a function have keyword arguments? + + >>> def f(x, y=0): + ... return x + y + + >>> has_kwargs(f) + True + """ + if sys.version_info[0] == 2: + spec = inspect.getargspec(f) + return bool(spec and (spec.keywords or spec.defaults)) + if sys.version_info[0] == 3: + spec = inspect.getfullargspec(f) + return bool(spec.defaults) + + +cpdef object isunary(object f): + """ + Does a function have only a single argument? + + >>> def f(x): + ... return x + + >>> isunary(f) + True + >>> isunary(lambda x, y: x + y) + False + """ + try: + if sys.version_info[0] == 2: + spec = inspect.getargspec(f) + if sys.version_info[0] == 3: + spec = inspect.getfullargspec(f) + return bool(spec and spec.varargs is None and not has_kwargs(f) + and len(spec.args) == 1) + except TypeError: + pass + return None # in Python < 3.4 builtins fail, return None + + cdef class c_memoize: property __doc__: def __get__(self): @@ -259,12 +315,9 @@ cdef class c_memoize: self.key = key try: - spec = inspect.getargspec(func) - self.may_have_kwargs = bool(not spec or spec.keywords or - spec.defaults) + self.may_have_kwargs = has_kwargs(func) # Is unary function (single arg, no variadic argument or keywords)? - self.is_unary = (spec and spec.varargs is None and - not self.may_have_kwargs and len(spec.args) == 1) + self.is_unary = isunary(func) except TypeError: self.is_unary = False self.may_have_kwargs = True diff --git a/cytoolz/tests/test_functoolz.py b/cytoolz/tests/test_functoolz.py index 7868cd3..951c8c3 100644 --- a/cytoolz/tests/test_functoolz.py +++ b/cytoolz/tests/test_functoolz.py @@ -314,7 +314,7 @@ def f(x, y=0): def test__num_required_args(): - assert _num_required_args(map) is None + assert _num_required_args(map) != 0 assert _num_required_args(lambda x: x) == 1 assert _num_required_args(lambda x, y: x) == 2 From 39b99caa03b2d002131ee94683cf5a8986d0ae04 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 2 Jul 2014 18:33:07 -0400 Subject: [PATCH 027/146] Add PyToolz mailing list to README.rst --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 49f41a6..ff7521e 100644 --- a/README.rst +++ b/README.rst @@ -66,6 +66,12 @@ Please take a look at our issue pages for `cytoolz `__ for contribution ideas. +Community +--------- + +See our `mailing list `__. +We're friendly. + .. |Build Status| image:: https://travis-ci.org/pytoolz/cytoolz.png :target: https://travis-ci.org/pytoolz/cytoolz .. |Version Status| image:: https://pypip.in/v/cytoolz/badge.png From 833070d725f42e0235ccdbe0b604c3dedff4868c Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 2 Jul 2014 18:50:14 -0400 Subject: [PATCH 028/146] Bump version to 0.7.0 (which will also run release tests) --- conda.yaml | 2 +- cytoolz/_version.py | 4 ++-- setup.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda.yaml b/conda.yaml index cd4c343..d710ff0 100644 --- a/conda.yaml +++ b/conda.yaml @@ -1,6 +1,6 @@ package: name: cytoolz - version: "0.6.1" + version: "0.7.0" build: number: {{environ.get('BINSTAR_BUILD', 1)}} diff --git a/cytoolz/_version.py b/cytoolz/_version.py index f84c9d8..52e1500 100644 --- a/cytoolz/_version.py +++ b/cytoolz/_version.py @@ -1,2 +1,2 @@ -__version__ = '0.6.2dev' -__toolz_version__ = '0.6.0' +__version__ = '0.7.0' +__toolz_version__ = '0.7.0' diff --git a/setup.py b/setup.py index 5f4e9c1..68501a7 100644 --- a/setup.py +++ b/setup.py @@ -99,7 +99,7 @@ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', - # 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.4', 'Topic :: Scientific/Engineering', 'Topic :: Scientific/Engineering :: Information Analysis', 'Topic :: Software Development', From cc33c427d95f9d7d5542da5f529ebf5146037030 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 2 Jul 2014 19:36:28 -0400 Subject: [PATCH 029/146] Don't test if `toolz.curried.curry` is curried. --- cytoolz/tests/test_curried_toolzlike.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cytoolz/tests/test_curried_toolzlike.py b/cytoolz/tests/test_curried_toolzlike.py index 333e229..2ebb99f 100644 --- a/cytoolz/tests/test_curried_toolzlike.py +++ b/cytoolz/tests/test_curried_toolzlike.py @@ -19,6 +19,8 @@ def test_toolzcurry_is_class(): def test_cytoolz_like_toolz(): for key, val in toolz.curried.__dict__.items(): if isinstance(val, toolz.curry): + if val.func is toolz.curry: # XXX: Python 3.4 work-around! + continue assert hasattr(cytoolz.curried, key), ( 'cytoolz.curried.%s does not exist' % key) assert isinstance(getattr(cytoolz.curried, key), cytoolz.curry), ( From 9c8a0635e8744a4b85c755fc5ed4d17681d1c588 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Fri, 4 Jul 2014 14:50:28 -0400 Subject: [PATCH 030/146] Bump to dev version --- cytoolz/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cytoolz/_version.py b/cytoolz/_version.py index 52e1500..4f7454d 100644 --- a/cytoolz/_version.py +++ b/cytoolz/_version.py @@ -1,2 +1,2 @@ -__version__ = '0.7.0' +__version__ = '0.7.1dev' __toolz_version__ = '0.7.0' From 55069a32974e8aa810ba736ab832d90c4e2251b8 Mon Sep 17 00:00:00 2001 From: Lars Buitinck Date: Tue, 26 Aug 2014 16:37:16 +0200 Subject: [PATCH 031/146] some extra C typing in functoolz.pyx --- cytoolz/functoolz.pyx | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index c44c30f..40cc896 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -120,7 +120,7 @@ def thread_last(val, *forms): known_numargs = {map: 2, filter: 2, reduce: 2, imap: 2, ifilter: 2} -cpdef object _num_required_args(object func): +cpdef Py_ssize_t _num_required_args(object func) except *: """ Number of args for func @@ -134,19 +134,21 @@ cpdef object _num_required_args(object func): ... return sum(args) >>> print(_num_required_args(bar)) - None + -1 """ + cdef Py_ssize_t num_defaults + if func in known_numargs: return known_numargs[func] try: spec = inspect.getargspec(func) if spec.varargs: - return None + return -1 num_defaults = len(spec.defaults) if spec.defaults else 0 return len(spec.args) - num_defaults except TypeError: pass - return None + return -1 cdef class curry: @@ -225,6 +227,7 @@ cdef class curry: def __call__(self, *args, **kwargs): cdef PyObject *obj + cdef Py_ssize_t required_args cdef object val if PyTuple_GET_SIZE(args) == 0: @@ -244,7 +247,7 @@ cdef class curry: PyErr_Clear() required_args = _num_required_args(self.func) # If there was a genuine TypeError - if required_args is None or len(args) < required_args: + if required_args == -1 or len(args) < required_args: return curry(self.func, *args, **kwargs) raise val @@ -285,10 +288,11 @@ cpdef object isunary(object f): >>> isunary(lambda x, y: x + y) False """ + cdef int major = sys.version_info[0] try: - if sys.version_info[0] == 2: + if major == 2: spec = inspect.getargspec(f) - if sys.version_info[0] == 3: + if major == 3: spec = inspect.getfullargspec(f) return bool(spec and spec.varargs is None and not has_kwargs(f) and len(spec.args) == 1) From a520aed839e7cdc04f712d0ff59de120f1e4aea3 Mon Sep 17 00:00:00 2001 From: Matthew Rocklin Date: Sat, 15 Nov 2014 10:24:03 -0800 Subject: [PATCH 032/146] binstar: add more platforms/versions --- .binstar.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.binstar.yml b/.binstar.yml index e371a15..63dabbc 100644 --- a/.binstar.yml +++ b/.binstar.yml @@ -3,9 +3,13 @@ platform: - linux-64 - linux-32 - osx-64 + - win-64 + - win-32 engine: - - python=2 - - python=3 + - python=2.6 + - python=2.7 + - python=3.3 + - python=3.4 script: - touch build.sh - conda build . From 0cee2edc977e0e0ce97165efaec07ace83c3a8b0 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 14 Dec 2014 17:25:17 -0600 Subject: [PATCH 033/146] Add `dissoc`, `tail`, and `getter`. Cytoolz once again matches toolz master --- cytoolz/__init__.pxd | 6 +-- cytoolz/curried.py | 2 + cytoolz/dicttoolz.pxd | 3 ++ cytoolz/dicttoolz.pyx | 20 +++++++++- cytoolz/functoolz.pyx | 6 +++ cytoolz/itertoolz.pxd | 17 +++++++++ cytoolz/itertoolz.pyx | 68 ++++++++++++++++++++++++++++++++- cytoolz/tests/test_dicttoolz.py | 15 +++++++- cytoolz/tests/test_itertoolz.py | 17 ++++++++- cytoolz/tests/test_none_safe.py | 9 +++++ 10 files changed, 154 insertions(+), 9 deletions(-) diff --git a/cytoolz/__init__.pxd b/cytoolz/__init__.pxd index 7e08db8..832bd66 100644 --- a/cytoolz/__init__.pxd +++ b/cytoolz/__init__.pxd @@ -2,7 +2,7 @@ from cytoolz.itertoolz cimport ( accumulate, c_merge_sorted, cons, count, drop, get, groupby, first, frequencies, interleave, interpose, isdistinct, isiterable, iterate, last, mapcat, nth, partition, partition_all, pluck, reduceby, remove, - rest, second, sliding_window, take, take_nth, unique, join) + rest, second, sliding_window, take, tail, take_nth, unique, join) from cytoolz.functoolz cimport ( @@ -11,8 +11,8 @@ from cytoolz.functoolz cimport ( from cytoolz.dicttoolz cimport ( - assoc, c_merge, c_merge_with, get_in, keyfilter, keymap, update_in, - valfilter, valmap) + assoc, c_merge, c_merge_with, dissoc, get_in, keyfilter, keymap, + update_in, valfilter, valmap) from cytoolz.recipes cimport countby, partitionby diff --git a/cytoolz/curried.py b/cytoolz/curried.py index 098ae68..43cb430 100644 --- a/cytoolz/curried.py +++ b/cytoolz/curried.py @@ -43,6 +43,7 @@ assoc = cytoolz.curry(assoc) cons = cytoolz.curry(cons) countby = cytoolz.curry(countby) +dissoc = cytoolz.curry(dissoc) do = cytoolz.curry(do) drop = cytoolz.curry(drop) filter = cytoolz.curry(filter) @@ -69,6 +70,7 @@ remove = cytoolz.curry(remove) sliding_window = cytoolz.curry(sliding_window) sorted = cytoolz.curry(sorted) +tail = cytoolz.curry(tail) take = cytoolz.curry(take) take_nth = cytoolz.curry(take_nth) unique = cytoolz.curry(unique) diff --git a/cytoolz/dicttoolz.pxd b/cytoolz/dicttoolz.pxd index 673d1fd..f0ff1e5 100644 --- a/cytoolz/dicttoolz.pxd +++ b/cytoolz/dicttoolz.pxd @@ -19,6 +19,9 @@ cpdef dict keyfilter(object predicate, dict d) cpdef dict assoc(dict d, object key, object value) +cpdef dict dissoc(dict d, object key) + + cpdef dict update_in(dict d, object keys, object func, object default=*) diff --git a/cytoolz/dicttoolz.pyx b/cytoolz/dicttoolz.pyx index 0b576eb..aab34f1 100644 --- a/cytoolz/dicttoolz.pyx +++ b/cytoolz/dicttoolz.pyx @@ -1,7 +1,7 @@ #cython: embedsignature=True from cpython.dict cimport (PyDict_Check, PyDict_GetItem, PyDict_Merge, PyDict_New, PyDict_Next, PyDict_SetItem, - PyDict_Update) + PyDict_Update, PyDict_DelItem) from cpython.exc cimport PyErr_Clear, PyErr_GivenExceptionMatches, PyErr_Occurred from cpython.list cimport PyList_Append, PyList_New from cpython.ref cimport PyObject @@ -11,7 +11,7 @@ from .cpython cimport PtrObject_GetItem __all__ = ['merge', 'merge_with', 'valmap', 'keymap', 'valfilter', 'keyfilter', - 'assoc', 'get_in', 'update_in'] + 'assoc', 'dissoc', 'get_in', 'update_in'] cdef dict c_merge(object dicts): @@ -215,6 +215,22 @@ cpdef dict assoc(dict d, object key, object value): return rv +cpdef dict dissoc(dict d, object key): + """ + Return a new dict with the given key removed. + + New dict has d[key] deleted. + Does not modify the initial dictionary. + + >>> dissoc({'x': 1, 'y': 2}, 'y') + {'x': 1} + """ + cdef dict rv + rv = d.copy() + PyDict_DelItem(rv, key) + return rv + + cpdef dict update_in(dict d, object keys, object func, object default=None): """ Update value in a (potentially) nested dictionary diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 40cc896..4663ac6 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -251,6 +251,9 @@ cdef class curry: return curry(self.func, *args, **kwargs) raise val + def __get__(self, instance, owner): + return curry(self, instance) + def __reduce__(self): return (curry, (self.func,), (self.args, self.keywords)) @@ -345,6 +348,9 @@ cdef class c_memoize: self.cache[key] = result return result + def __get__(self, instance, owner): + return curry(self, instance) + cpdef object memoize(object func=None, object cache=None, object key=None): """ diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index d7c2829..a21e779 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -149,6 +149,23 @@ cdef class _pluck_list_default: cpdef object pluck(object ind, object seqs, object default=*) + +cdef class _getter_index: + cdef object ind + + +cdef class _getter_list: + cdef list ind + cdef Py_ssize_t n + + +cdef class _getter_null: + pass + + +cpdef object getter(object index) + + cpdef object join(object leftkey, object leftseq, object rightkey, object rightseq, object left_default=*, diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index b7f5ad5..84f20ed 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -11,6 +11,7 @@ from cpython.tuple cimport PyTuple_GetSlice, PyTuple_New, PyTuple_SET_ITEM # Locally defined bindings that differ from `cython.cpython` bindings from .cpython cimport PtrIter_Next, PtrObject_GetItem +from collections import deque from heapq import heapify, heappop, heapreplace from itertools import chain, islice from operator import itemgetter @@ -22,7 +23,7 @@ __all__ = ['remove', 'accumulate', 'groupby', 'merge_sorted', 'interleave', 'first', 'second', 'nth', 'last', 'get', 'concat', 'concatv', 'mapcat', 'cons', 'interpose', 'frequencies', 'reduceby', 'iterate', 'sliding_window', 'partition', 'partition_all', 'count', 'pluck', - 'join'] + 'join', 'tail'] concatv = chain @@ -472,16 +473,40 @@ cpdef object take(Py_ssize_t n, object seq): >>> list(take(2, [10, 20, 30, 40, 50])) [10, 20] + + See Also: + drop + tail """ return islice(seq, n) +cpdef object tail(Py_ssize_t n, object seq): + """ + The last n elements of a sequence + + >>> tail(2, [10, 20, 30, 40, 50]) + [40, 50] + + See Also: + drop + take + """ + if PySequence_Check(seq): + return seq[-n:] + return tuple(deque(seq, n)) + + cpdef object drop(Py_ssize_t n, object seq): """ The sequence following the first n elements >>> list(drop(2, [10, 20, 30, 40, 50])) [30, 40, 50] + + See Also: + take + tail """ if n < 0: raise ValueError('n argument for drop() must be non-negative') @@ -1108,6 +1133,47 @@ cpdef object pluck(object ind, object seqs, object default=no_default): return _pluck_index_default(ind, seqs, default) +cdef class _getter_index: + def __cinit__(self, object ind): + self.ind = ind + + def __call__(self, object seq): + return seq[self.ind] + + +cdef class _getter_list: + def __cinit__(self, list ind not None): + self.ind = ind + self.n = len(ind) + + def __call__(self, object seq): + cdef Py_ssize_t i + cdef tuple result + cdef object val + result = PyTuple_New(self.n) + for i, val in enumerate(self.ind): + val = seq[val] + Py_INCREF(val) + PyTuple_SET_ITEM(result, i, val) + return result + + +cdef class _getter_null: + def __call__(self, object seq): + return () + + +# TODO: benchmark getters (and compare against itemgetter) +cpdef object getter(object index): + if isinstance(index, list): + if PyList_GET_SIZE(index) == 0: + return _getter_null() + elif PyList_GET_SIZE(index) < 10: + return _getter_list(index) + return itemgetter(*index) + return _getter_index(index) + + cpdef object join(object leftkey, object leftseq, object rightkey, object rightseq, object left_default=no_default, diff --git a/cytoolz/tests/test_dicttoolz.py b/cytoolz/tests/test_dicttoolz.py index f4c36db..b988db1 100644 --- a/cytoolz/tests/test_dicttoolz.py +++ b/cytoolz/tests/test_dicttoolz.py @@ -1,6 +1,6 @@ from cytoolz.utils import raises from cytoolz.dicttoolz import (merge, merge_with, valmap, keymap, update_in, - assoc, keyfilter, valfilter) + assoc, dissoc, keyfilter, valfilter) inc = lambda x: x + 1 @@ -64,6 +64,19 @@ def test_assoc(): assert d is oldd +def test_dissoc(): + assert dissoc({"a": 1}, "a") == {} + assert dissoc({"a": 1, "b": 2}, "a") == {"b": 2} + assert dissoc({"a": 1, "b": 2}, "b") == {"a": 1} + + # Verify immutability: + d = {'x': 1} + oldd = d + d2 = dissoc(d, 'x') + assert d is oldd + assert d2 is not oldd + + def test_update_in(): assert update_in({"a": 0}, ["a"], inc) == {"a": 1} assert update_in({"a": 0, "b": 1}, ["b"], str) == {"a": 0, "b": "1"} diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index 5420dcd..47920d6 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -4,9 +4,9 @@ from functools import partial from cytoolz.itertoolz import (remove, groupby, merge_sorted, concat, concatv, interleave, unique, - isiterable, + isiterable, getter, mapcat, isdistinct, first, second, - nth, take, drop, interpose, get, + nth, take, tail, drop, interpose, get, rest, last, cons, frequencies, reduceby, iterate, accumulate, sliding_window, count, partition, @@ -140,6 +140,12 @@ def test_take(): assert list(take(2, (3, 2, 1))) == list((3, 2)) +def test_tail(): + assert list(tail(3, 'ABCDE')) == list('CDE') + assert list(tail(3, iter('ABCDE'))) == list('CDE') + assert list(tail(2, (3, 2, 1))) == list((2, 1)) + + def test_drop(): assert list(drop(3, 'ABCDE')) == list('DE') assert list(drop(1, (3, 2, 1))) == list((2, 1)) @@ -160,6 +166,7 @@ def test_get(): assert get([0, 2], 'AB', 'C') == ('A', 'C') assert get([0], 'AB') == ('A',) + assert get([], 'AB') == () assert raises(IndexError, lambda: get(10, 'ABC')) assert raises(KeyError, lambda: get(10, {'a': 1})) @@ -322,6 +329,12 @@ def addpair(pair): assert result == expected +def test_getter(): + assert getter(0)('Alice') == 'A' + assert getter([0])('Alice') == ('A',) + assert getter([])('Alice') == () + + def test_key_as_getter(): squares = [(i, i**2) for i in range(5)] pows = [(i, i**2, i**3) for i in range(5)] diff --git a/cytoolz/tests/test_none_safe.py b/cytoolz/tests/test_none_safe.py index b129447..4c965b1 100644 --- a/cytoolz/tests/test_none_safe.py +++ b/cytoolz/tests/test_none_safe.py @@ -43,6 +43,9 @@ def test_dicttoolz(): assert raises((TypeError, AttributeError), lambda: assoc(None, 1, 2)) tested.append('assoc') + assert raises((TypeError, AttributeError), lambda: dissoc(None, 1)) + tested.append('dissoc') + # XXX assert (raises(TypeError, lambda: get_in(None, {})) or get_in(None, {}) is None) @@ -246,6 +249,12 @@ def test_itertoolz(): assert raises(TypeError, lambda: list(take(1, None))) tested.append('take') + # XXX + assert (raises(TypeError, lambda: list(tail(None, [1, 2])) == [1, 2]) or + list(tail(None, [1, 2])) == [1, 2]) + assert raises(TypeError, lambda: list(tail(1, None))) + tested.append('tail') + # XXX assert (raises(TypeError, lambda: list(take_nth(None, [1, 2]))) or list(take_nth(None, [1, 2])) == [1, 2]) From e81f1314f2071a2fbce84d3101b39e2f1eb30613 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 20 Dec 2014 13:31:43 -0600 Subject: [PATCH 034/146] Fix `nth` for negative indexes on iterators. Needs to raise ValueError. --- cytoolz/itertoolz.pyx | 2 ++ cytoolz/tests/test_itertoolz.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index b7f5ad5..a86269c 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -537,6 +537,8 @@ cpdef object nth(Py_ssize_t n, object seq): """ if PySequence_Check(seq): return seq[n] + if n < 0: + raise ValueError('"n" must be positive when indexing an iterator') seq = iter(seq) while n > 0: n -= 1 diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index 5420dcd..52008e8 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -110,6 +110,8 @@ def test_nth(): assert nth(1, (3, 2, 1)) == 2 assert nth(0, {'foo': 'bar'}) == 'foo' assert raises(StopIteration, lambda: nth(10, {10: 'foo'})) + assert nth(-2, 'ABCDE') == 'D' + assert raises(ValueError, lambda: nth(-2, iter('ABCDE'))) def test_first(): From e749d8351a161e2c1c9cbbc9c6e973fa61cda9cf Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 20 Dec 2014 14:50:24 -0600 Subject: [PATCH 035/146] Add itemmap and itemfilter. Update reduceby to accept callable initial value. This brings cytoolz up to date with toolz. These additions weren't benchmarked. --- cytoolz/__init__.pxd | 2 +- cytoolz/curried.py | 2 + cytoolz/dicttoolz.pxd | 6 +++ cytoolz/dicttoolz.pyx | 72 ++++++++++++++++++++++++++++++++- cytoolz/itertoolz.pyx | 26 ++++++++++-- cytoolz/tests/test_dicttoolz.py | 12 +++++- cytoolz/tests/test_itertoolz.py | 9 +++++ cytoolz/tests/test_none_safe.py | 9 +++++ 8 files changed, 130 insertions(+), 8 deletions(-) diff --git a/cytoolz/__init__.pxd b/cytoolz/__init__.pxd index 832bd66..ba716bb 100644 --- a/cytoolz/__init__.pxd +++ b/cytoolz/__init__.pxd @@ -12,7 +12,7 @@ from cytoolz.functoolz cimport ( from cytoolz.dicttoolz cimport ( assoc, c_merge, c_merge_with, dissoc, get_in, keyfilter, keymap, - update_in, valfilter, valmap) + itemfilter, itemmap, update_in, valfilter, valmap) from cytoolz.recipes cimport countby, partitionby diff --git a/cytoolz/curried.py b/cytoolz/curried.py index 43cb430..bc08af8 100644 --- a/cytoolz/curried.py +++ b/cytoolz/curried.py @@ -52,6 +52,8 @@ groupby = cytoolz.curry(groupby) interleave = cytoolz.curry(interleave) interpose = cytoolz.curry(interpose) +itemfilter = cytoolz.curry(itemfilter) +itemmap = cytoolz.curry(itemmap) iterate = cytoolz.curry(iterate) join = cytoolz.curry(join) keyfilter = cytoolz.curry(keyfilter) diff --git a/cytoolz/dicttoolz.pxd b/cytoolz/dicttoolz.pxd index f0ff1e5..bf798f9 100644 --- a/cytoolz/dicttoolz.pxd +++ b/cytoolz/dicttoolz.pxd @@ -10,12 +10,18 @@ cpdef dict valmap(object func, dict d) cpdef dict keymap(object func, dict d) +cpdef dict itemmap(object func, dict d) + + cpdef dict valfilter(object predicate, dict d) cpdef dict keyfilter(object predicate, dict d) +cpdef dict itemfilter(object predicate, dict d) + + cpdef dict assoc(dict d, object key, object value) diff --git a/cytoolz/dicttoolz.pyx b/cytoolz/dicttoolz.pyx index aab34f1..d57ecd6 100644 --- a/cytoolz/dicttoolz.pyx +++ b/cytoolz/dicttoolz.pyx @@ -10,8 +10,8 @@ from cpython.ref cimport PyObject from .cpython cimport PtrObject_GetItem -__all__ = ['merge', 'merge_with', 'valmap', 'keymap', 'valfilter', 'keyfilter', - 'assoc', 'dissoc', 'get_in', 'update_in'] +__all__ = ['merge', 'merge_with', 'valmap', 'keymap', 'itemmap', 'valfilter', + 'keyfilter', 'itemfilter', 'assoc', 'dissoc', 'get_in', 'update_in'] cdef dict c_merge(object dicts): @@ -94,6 +94,7 @@ cpdef dict valmap(object func, dict d): See Also: keymap + itemmap """ cdef: dict rv @@ -121,6 +122,7 @@ cpdef dict keymap(object func, dict d): See Also: valmap + itemmap """ cdef: dict rv @@ -138,6 +140,36 @@ cpdef dict keymap(object func, dict d): return rv +cpdef dict itemmap(object func, dict d): + """ + Apply function to items of dictionary + + >>> accountids = {"Alice": 10, "Bob": 20} + >>> itemmap(reversed, accountids) # doctest: +SKIP + {10: "Alice", 20: "Bob"} + + See Also: + keymap + valmap + """ + cdef: + dict rv + object newk, newv + Py_ssize_t pos + PyObject *k + PyObject *v + + if d is None: + raise TypeError("expected dict, got None") + + rv = PyDict_New() + pos = 0 + while PyDict_Next(d, &pos, &k, &v): + newk, newv = func((k, v)) + PyDict_SetItem(rv, newk, newv) + return rv + + cpdef dict valfilter(object predicate, dict d): """ Filter items in dictionary by value @@ -149,6 +181,7 @@ cpdef dict valfilter(object predicate, dict d): See Also: keyfilter + itemfilter valmap """ cdef: @@ -179,6 +212,7 @@ cpdef dict keyfilter(object predicate, dict d): See Also: valfilter + itemfilter keymap """ cdef: @@ -198,6 +232,40 @@ cpdef dict keyfilter(object predicate, dict d): return rv +cpdef dict itemfilter(object predicate, dict d): + """ + Filter items in dictionary by item + + >>> def isvalid(item): + ... k, v = item + ... return k % 2 == 0 and v < 4 + + >>> d = {1: 2, 2: 3, 3: 4, 4: 5} + >>> itemfilter(isvalid, d) + {2: 3} + + See Also: + keyfilter + valfilter + itemmap + """ + cdef: + dict rv + Py_ssize_t pos + PyObject *k + PyObject *v + + if d is None: + raise TypeError("expected dict, got None") + + rv = PyDict_New() + pos = 0 + while PyDict_Next(d, &pos, &k, &v): + if predicate((k, v)): + PyDict_SetItem(rv, k, v) + return rv + + cpdef dict assoc(dict d, object key, object value): """ Return a new dict with new key value pair diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 17f9112..d9e1e32 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -754,12 +754,14 @@ cpdef dict frequencies(object seq): cdef inline object _reduceby_core(dict d, object key, object item, object binop, - object init, bint skip_init): + object init, bint skip_init, bint call_init): cdef PyObject *obj = PyDict_GetItem(d, key) if obj is not NULL: PyDict_SetItem(d, key, binop(obj, item)) elif skip_init: PyDict_SetItem(d, key, item) + elif call_init: + PyDict_SetItem(d, key, binop(init(), item)) else: PyDict_SetItem(d, key, binop(init, item)) @@ -784,6 +786,10 @@ cpdef dict reduceby(object key, object binop, object seq, object init=no_default operate in much less space. This makes it suitable for larger datasets that do not fit comfortably in memory + The ``init`` keyword argument is the default initialization of the + reduction. This can be either a constant value like ``0`` or a callable + like ``lambda : 0`` as might be used in ``defaultdict``. + Simple Examples --------------- @@ -810,15 +816,27 @@ cpdef dict reduceby(object key, object binop, object seq, object init=no_default ... lambda acc, x: acc + x['cost'], ... projects, 0) {'CA': 1200000, 'IL': 2100000} + + Example Using ``init`` + ---------------------- + + >>> def set_add(s, i): + ... s.add(i) + ... return s + + >>> reduceby(iseven, set_add, [1, 2, 3, 4, 1, 2, 3], set) # doctest: +SKIP + {True: set([2, 4]), + False: set([1, 3])} """ cdef dict d = {} cdef object item, keyval cdef Py_ssize_t i, N cdef bint skip_init = init is no_default + cdef bint call_init = callable(init) if callable(key): for item in seq: keyval = key(item) - _reduceby_core(d, keyval, item, binop, init, skip_init) + _reduceby_core(d, keyval, item, binop, init, skip_init, call_init) elif isinstance(key, list): N = PyList_GET_SIZE(key) for item in seq: @@ -828,11 +846,11 @@ cpdef dict reduceby(object key, object binop, object seq, object init=no_default val = item[val] Py_INCREF(val) PyTuple_SET_ITEM(keyval, i, val) - _reduceby_core(d, keyval, item, binop, init, skip_init) + _reduceby_core(d, keyval, item, binop, init, skip_init, call_init) else: for item in seq: keyval = item[key] - _reduceby_core(d, keyval, item, binop, init, skip_init) + _reduceby_core(d, keyval, item, binop, init, skip_init, call_init) return d diff --git a/cytoolz/tests/test_dicttoolz.py b/cytoolz/tests/test_dicttoolz.py index b988db1..e607c26 100644 --- a/cytoolz/tests/test_dicttoolz.py +++ b/cytoolz/tests/test_dicttoolz.py @@ -1,6 +1,7 @@ from cytoolz.utils import raises from cytoolz.dicttoolz import (merge, merge_with, valmap, keymap, update_in, - assoc, dissoc, keyfilter, valfilter) + assoc, dissoc, keyfilter, valfilter, itemmap, + itemfilter) inc = lambda x: x + 1 @@ -44,6 +45,10 @@ def test_keymap(): assert keymap(inc, {1: 1, 2: 2}) == {2: 1, 3: 2} +def test_itemmap(): + assert itemmap(reversed, {1: 2, 2: 4}) == {2: 1, 4: 2} + + def test_valfilter(): assert valfilter(iseven, {1: 2, 2: 3}) == {1: 2} @@ -52,6 +57,11 @@ def test_keyfilter(): assert keyfilter(iseven, {1: 2, 2: 3}) == {2: 3} +def test_itemfilter(): + assert itemfilter(lambda item: iseven(item[0]), {1: 2, 2: 3}) == {2: 3} + assert itemfilter(lambda item: iseven(item[1]), {1: 2, 2: 3}) == {1: 2} + + def test_assoc(): assert assoc({}, "a", 1) == {"a": 1} assert assoc({"a": 1}, "a", 3) == {"a": 3} diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index 92f1493..a057606 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -251,6 +251,15 @@ def test_reduce_by_init(): assert reduceby(iseven, add, [1, 2, 3, 4]) == {True: 2 + 4, False: 1 + 3} +def test_reduce_by_callable_default(): + def set_add(s, i): + s.add(i) + return s + + assert reduceby(iseven, set_add, [1, 2, 3, 4, 1, 2], set) == \ + {True: set([2, 4]), False: set([1, 3])} + + def test_iterate(): assert list(itertools.islice(iterate(inc, 0), 0, 5)) == [0, 1, 2, 3, 4] assert list(take(4, iterate(double, 1))) == [1, 2, 4, 8] diff --git a/cytoolz/tests/test_none_safe.py b/cytoolz/tests/test_none_safe.py index 4c965b1..a76944f 100644 --- a/cytoolz/tests/test_none_safe.py +++ b/cytoolz/tests/test_none_safe.py @@ -91,6 +91,15 @@ def test_dicttoolz(): assert raises(TypeError, lambda: valmap(identity, None)) tested.append('valmap') + assert (raises(TypeError, lambda: itemmap(None, {1: 2})) or + itemmap(None, {1: 2}) == {1: (2,)}) + assert raises(TypeError, lambda: itemmap(identity, None)) + tested.append('itemmap') + + assert raises(TypeError, lambda: itemfilter(None, {1: 2})) + assert raises(TypeError, lambda: itemfilter(identity, None)) + tested.append('itemfilter') + s1 = set(tested) s2 = set(cytoolz.dicttoolz.__all__) assert s1 == s2, '%s not tested for being None-safe' % ', '.join(s2 - s1) From 494257535c40e13a3275dab27124f4ba86d8b9db Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 20 Dec 2014 16:41:03 -0600 Subject: [PATCH 036/146] Update version (0.7.1 was just released) --- conda.yaml | 2 +- cytoolz/_version.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conda.yaml b/conda.yaml index d710ff0..2e8a27e 100644 --- a/conda.yaml +++ b/conda.yaml @@ -1,6 +1,6 @@ package: name: cytoolz - version: "0.7.0" + version: "0.7.1" build: number: {{environ.get('BINSTAR_BUILD', 1)}} diff --git a/cytoolz/_version.py b/cytoolz/_version.py index 4f7454d..f33da1a 100644 --- a/cytoolz/_version.py +++ b/cytoolz/_version.py @@ -1,2 +1,2 @@ -__version__ = '0.7.1dev' -__toolz_version__ = '0.7.0' +__version__ = '0.7.2dev' +__toolz_version__ = '0.7.1' From f384194692c8b2169f95f9280194ea70ccffd788 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 12:24:46 -0500 Subject: [PATCH 037/146] Ship tests with code --- cytoolz/tests/__init__.py | 0 setup.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 cytoolz/tests/__init__.py diff --git a/cytoolz/tests/__init__.py b/cytoolz/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py index 68501a7..e30040a 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,7 @@ maintainer='Erik Welch', maintainer_email='erik.n.welch@gmail.com', license = 'BSD', - packages=['cytoolz'], + packages=['cytoolz', 'cytoolz.tests'], package_data={'cytoolz': ['*.pxd']}, # include_package_data = True, keywords=('functional utility itertools functools iterator generator ' From 5ba0f43b2014af85f43f76dbdc4bca16de3ec4cb Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 12:24:58 -0500 Subject: [PATCH 038/146] Ignore build output and build.sh --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 1f543a6..022ea35 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ *.pyc *.swp *.so +build/ +dist/ +build.sh From c25d1b322e2e54a819da84bc9154730d47770e4b Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 12:44:30 -0500 Subject: [PATCH 039/146] Use package_data to ship tests --- MANIFEST.in | 1 + conda.yaml | 12 ++++++++++-- setup.py | 4 ++-- 3 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..d0ccc69 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include cytoolz/tests/*.py diff --git a/conda.yaml b/conda.yaml index 2e8a27e..15a7b94 100644 --- a/conda.yaml +++ b/conda.yaml @@ -4,8 +4,7 @@ package: build: number: {{environ.get('BINSTAR_BUILD', 1)}} - script: - - cd $RECIPE_DIR + script: - $PYTHON setup.py install --with-cython requirements: @@ -18,6 +17,15 @@ requirements: run: - python +test: + requires: + - pytest + - nose + imports: + - cytoolz + commands: + - py.test -x --doctest-modules --pyargs cytoolz + about: home: http://toolz.readthedocs.org/ license: BSD diff --git a/setup.py b/setup.py index e30040a..897ce01 100644 --- a/setup.py +++ b/setup.py @@ -79,8 +79,8 @@ maintainer='Erik Welch', maintainer_email='erik.n.welch@gmail.com', license = 'BSD', - packages=['cytoolz', 'cytoolz.tests'], - package_data={'cytoolz': ['*.pxd']}, + packages=['cytoolz'], + package_data={'cytoolz': ['*.pxd', 'tests/*.py']}, # include_package_data = True, keywords=('functional utility itertools functools iterator generator ' 'curry memoize lazy streaming bigdata cython toolz cytoolz'), From 696837fa06cc4c3988d0e902036ab5629bfdf85a Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 12:46:13 -0500 Subject: [PATCH 040/146] No need for build.sh --- .binstar.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.binstar.yml b/.binstar.yml index 63dabbc..f3e44ec 100644 --- a/.binstar.yml +++ b/.binstar.yml @@ -11,7 +11,6 @@ engine: - python=3.3 - python=3.4 script: - - touch build.sh - conda build . build_targets: files: conda From a53b8e8c5c0f3da11b7467bdd8e431f45b89c815 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 12:56:41 -0500 Subject: [PATCH 041/146] Move to own dir --- .binstar.yml | 2 +- .gitignore | 1 - conda.recipe/bld.bat | 2 ++ conda.recipe/build.sh | 2 ++ conda.yaml => conda.recipe/meta.yaml | 0 5 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 conda.recipe/bld.bat create mode 100755 conda.recipe/build.sh rename conda.yaml => conda.recipe/meta.yaml (100%) diff --git a/.binstar.yml b/.binstar.yml index f3e44ec..257744c 100644 --- a/.binstar.yml +++ b/.binstar.yml @@ -11,7 +11,7 @@ engine: - python=3.3 - python=3.4 script: - - conda build . + - conda build conda.recipe build_targets: files: conda channels: main diff --git a/.gitignore b/.gitignore index 022ea35..7bde555 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,3 @@ *.so build/ dist/ -build.sh diff --git a/conda.recipe/bld.bat b/conda.recipe/bld.bat new file mode 100644 index 0000000..5cd8f21 --- /dev/null +++ b/conda.recipe/bld.bat @@ -0,0 +1,2 @@ +cd %RECIPE_DIR%\.. +%PYTHON% setup.py install --with-cython diff --git a/conda.recipe/build.sh b/conda.recipe/build.sh new file mode 100755 index 0000000..46e7209 --- /dev/null +++ b/conda.recipe/build.sh @@ -0,0 +1,2 @@ +cd $RECIPE_DIR/.. +$PYTHON setup.py install --with-cython diff --git a/conda.yaml b/conda.recipe/meta.yaml similarity index 100% rename from conda.yaml rename to conda.recipe/meta.yaml From 0320df8ec5610f4b646c7569a017cf055ecc366b Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 12:57:41 -0500 Subject: [PATCH 042/146] No script field --- conda.recipe/meta.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 15a7b94..4999fef 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -4,8 +4,6 @@ package: build: number: {{environ.get('BINSTAR_BUILD', 1)}} - script: - - $PYTHON setup.py install --with-cython requirements: build: From de626fb683a5d76a13b5005672c3de7cb3f1c335 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 13:00:04 -0500 Subject: [PATCH 043/146] we need toolz to test --- conda.recipe/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 4999fef..bbbbc12 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -19,6 +19,7 @@ test: requires: - pytest - nose + - toolz imports: - cytoolz commands: From 395e42accd1c771ce94738bf05e609fec99ed4bf Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 13:08:29 -0500 Subject: [PATCH 044/146] Make work with py3.x --- cytoolz/tests/__init__.py | 0 cytoolz/tests/test_curried_toolzlike.py | 2 +- cytoolz/tests/test_dev_skip_test.py | 2 +- cytoolz/tests/test_docstrings.py | 2 +- cytoolz/tests/test_embedded_sigs.py | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 cytoolz/tests/__init__.py diff --git a/cytoolz/tests/__init__.py b/cytoolz/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/cytoolz/tests/test_curried_toolzlike.py b/cytoolz/tests/test_curried_toolzlike.py index 2ebb99f..aa6fb62 100644 --- a/cytoolz/tests/test_curried_toolzlike.py +++ b/cytoolz/tests/test_curried_toolzlike.py @@ -3,7 +3,7 @@ import toolz import toolz.curried import types -from dev_skip_test import dev_skip_test +from cytoolz.tests.dev_skip_test import dev_skip_test # Note that the tests in this file assume `toolz.curry` is a class, but we diff --git a/cytoolz/tests/test_dev_skip_test.py b/cytoolz/tests/test_dev_skip_test.py index 0c25271..2a63cb9 100644 --- a/cytoolz/tests/test_dev_skip_test.py +++ b/cytoolz/tests/test_dev_skip_test.py @@ -1,4 +1,4 @@ -from dev_skip_test import istest, nottest, dev_skip_test +from cytoolz.tests.dev_skip_test import istest, nottest, dev_skip_test d = {} diff --git a/cytoolz/tests/test_docstrings.py b/cytoolz/tests/test_docstrings.py index 4be663c..030e093 100644 --- a/cytoolz/tests/test_docstrings.py +++ b/cytoolz/tests/test_docstrings.py @@ -4,7 +4,7 @@ from cytoolz import curry, identity, keyfilter, valfilter, merge_with from cytoolz.utils import raises -from dev_skip_test import dev_skip_test +from cytoolz.tests.dev_skip_test import dev_skip_test # `cytoolz` functions for which "# doctest: +SKIP" were added. diff --git a/cytoolz/tests/test_embedded_sigs.py b/cytoolz/tests/test_embedded_sigs.py index d2cd69f..1211f47 100644 --- a/cytoolz/tests/test_embedded_sigs.py +++ b/cytoolz/tests/test_embedded_sigs.py @@ -4,7 +4,7 @@ from types import BuiltinFunctionType from cytoolz import curry, identity, keyfilter, valfilter, merge_with -from dev_skip_test import dev_skip_test +from cytoolz.tests.dev_skip_test import dev_skip_test @curry From d458cd19bfc879ba17c6fe59facc817f91a82f25 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 13:13:07 -0500 Subject: [PATCH 045/146] Need init for dev skip test --- cytoolz/tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 cytoolz/tests/__init__.py diff --git a/cytoolz/tests/__init__.py b/cytoolz/tests/__init__.py new file mode 100644 index 0000000..e69de29 From c85966aaa649b8020e0fbf3e76b8fff519e931ff Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 13:38:56 -0500 Subject: [PATCH 046/146] Do not include cython files --- conda.recipe/bld.bat | 2 +- conda.recipe/build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda.recipe/bld.bat b/conda.recipe/bld.bat index 5cd8f21..a8cb1eb 100644 --- a/conda.recipe/bld.bat +++ b/conda.recipe/bld.bat @@ -1,2 +1,2 @@ cd %RECIPE_DIR%\.. -%PYTHON% setup.py install --with-cython +%PYTHON% setup.py install diff --git a/conda.recipe/build.sh b/conda.recipe/build.sh index 46e7209..abebc13 100755 --- a/conda.recipe/build.sh +++ b/conda.recipe/build.sh @@ -1,2 +1,2 @@ cd $RECIPE_DIR/.. -$PYTHON setup.py install --with-cython +$PYTHON setup.py install From e5417097ca3747a67fc6283b64b3a6ae57dd859c Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 13:43:10 -0500 Subject: [PATCH 047/146] Remove __init__.py --- cytoolz/tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 cytoolz/tests/__init__.py diff --git a/cytoolz/tests/__init__.py b/cytoolz/tests/__init__.py deleted file mode 100644 index e69de29..0000000 From b3e142248adc2235864becf8abf364210a5fe1c4 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 13:43:58 -0500 Subject: [PATCH 048/146] relative import --- cytoolz/tests/test_curried_toolzlike.py | 2 +- cytoolz/tests/test_dev_skip_test.py | 2 +- cytoolz/tests/test_docstrings.py | 2 +- cytoolz/tests/test_embedded_sigs.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cytoolz/tests/test_curried_toolzlike.py b/cytoolz/tests/test_curried_toolzlike.py index aa6fb62..2ebb99f 100644 --- a/cytoolz/tests/test_curried_toolzlike.py +++ b/cytoolz/tests/test_curried_toolzlike.py @@ -3,7 +3,7 @@ import toolz import toolz.curried import types -from cytoolz.tests.dev_skip_test import dev_skip_test +from dev_skip_test import dev_skip_test # Note that the tests in this file assume `toolz.curry` is a class, but we diff --git a/cytoolz/tests/test_dev_skip_test.py b/cytoolz/tests/test_dev_skip_test.py index 2a63cb9..0c25271 100644 --- a/cytoolz/tests/test_dev_skip_test.py +++ b/cytoolz/tests/test_dev_skip_test.py @@ -1,4 +1,4 @@ -from cytoolz.tests.dev_skip_test import istest, nottest, dev_skip_test +from dev_skip_test import istest, nottest, dev_skip_test d = {} diff --git a/cytoolz/tests/test_docstrings.py b/cytoolz/tests/test_docstrings.py index 030e093..4be663c 100644 --- a/cytoolz/tests/test_docstrings.py +++ b/cytoolz/tests/test_docstrings.py @@ -4,7 +4,7 @@ from cytoolz import curry, identity, keyfilter, valfilter, merge_with from cytoolz.utils import raises -from cytoolz.tests.dev_skip_test import dev_skip_test +from dev_skip_test import dev_skip_test # `cytoolz` functions for which "# doctest: +SKIP" were added. diff --git a/cytoolz/tests/test_embedded_sigs.py b/cytoolz/tests/test_embedded_sigs.py index 1211f47..d2cd69f 100644 --- a/cytoolz/tests/test_embedded_sigs.py +++ b/cytoolz/tests/test_embedded_sigs.py @@ -4,7 +4,7 @@ from types import BuiltinFunctionType from cytoolz import curry, identity, keyfilter, valfilter, merge_with -from cytoolz.tests.dev_skip_test import dev_skip_test +from dev_skip_test import dev_skip_test @curry From c54aaedc0f1d0caca7093987e3fd497a6c29f2b1 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 13:47:54 -0500 Subject: [PATCH 049/146] Add myself to authors.md --- AUTHORS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AUTHORS.md b/AUTHORS.md index 8a305fe..76cb1d2 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -7,3 +7,5 @@ Lars Buitinck [@larsmans](http://github.com/la [Thouis (Ray) Jones](http://people.seas.harvard.edu/~thouis) [@thouis](https://github.com/thouis/) scoder [@scoder](https://github.com/scoder/) + +Phillip Cloud [@cpcloud](https://github.com/cpcloud) From c7320b49ee5c84088a7035d91fefa2b93f521dfe Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 14:14:01 -0500 Subject: [PATCH 050/146] Windoze --- .gitignore | 1 + conda.recipe/meta.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 7bde555..09be199 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ *.pyc *.swp *.so +*.pyd build/ dist/ diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index bbbbc12..b6869d3 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -11,9 +11,11 @@ requirements: - python - cython - toolz + - libpython # [win] run: - python + - libpython # [win] test: requires: From 024a03df063a5d9aa7d41d08589d1f3700ca8ca3 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 14:20:49 -0500 Subject: [PATCH 051/146] Replace crlf with lf for win compat in py26 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 897ce01..127e31c 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,8 @@ info = {} filename = os.path.join('cytoolz', '_version.py') -exec(compile(open(filename, "rb").read(), filename, 'exec'), info) +exec(compile(open(filename, "rb").read().replace('\r\n', '\n'), + filename, 'exec'), info) VERSION = info['__version__'] try: From c250700b83a0dec6387f04c5b061d3e5e0677edf Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 14:28:33 -0500 Subject: [PATCH 052/146] Need bytes for py3.x --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 127e31c..84f45f7 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ info = {} filename = os.path.join('cytoolz', '_version.py') -exec(compile(open(filename, "rb").read().replace('\r\n', '\n'), +exec(compile(open(filename, "rb").read().replace(b'\r\n', b'\n'), filename, 'exec'), info) VERSION = info['__version__'] From 9102d6782a2155f3846a1e7d608281fdaae38d33 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 15:29:12 -0500 Subject: [PATCH 053/146] Move out of main --- setup.py | 92 +++++++++++++++++++++++++++----------------------------- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/setup.py b/setup.py index 84f45f7..6f5a1b5 100644 --- a/setup.py +++ b/setup.py @@ -63,50 +63,48 @@ if use_cython: ext_modules = cythonize(ext_modules) - -if __name__ == '__main__': - setup( - name='cytoolz', - version=VERSION, - description=('Cython implementation of Toolz: ' - 'High performance functional utilities'), - ext_modules=ext_modules, - long_description=(open('README.rst').read() - if os.path.exists('README.rst') - else ''), - url='https://github.com/pytoolz/cytoolz', - author='https://raw.github.com/pytoolz/cytoolz/master/AUTHORS.md', - author_email='erik.n.welch@gmail.com', - maintainer='Erik Welch', - maintainer_email='erik.n.welch@gmail.com', - license = 'BSD', - packages=['cytoolz'], - package_data={'cytoolz': ['*.pxd', 'tests/*.py']}, - # include_package_data = True, - keywords=('functional utility itertools functools iterator generator ' - 'curry memoize lazy streaming bigdata cython toolz cytoolz'), - classifiers = [ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Intended Audience :: Education', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Operating System :: OS Independent', - 'Programming Language :: Cython', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Topic :: Scientific/Engineering', - 'Topic :: Scientific/Engineering :: Information Analysis', - 'Topic :: Software Development', - 'Topic :: Software Development :: Libraries', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Utilities', - ], - # zip_safe=False - ) +setup( + name='cytoolz', + version=VERSION, + description=('Cython implementation of Toolz: ' + 'High performance functional utilities'), + ext_modules=ext_modules, + long_description=(open('README.rst').read() + if os.path.exists('README.rst') + else ''), + url='https://github.com/pytoolz/cytoolz', + author='https://raw.github.com/pytoolz/cytoolz/master/AUTHORS.md', + author_email='erik.n.welch@gmail.com', + maintainer='Erik Welch', + maintainer_email='erik.n.welch@gmail.com', + license = 'BSD', + packages=['cytoolz'], + package_data={'cytoolz': ['*.pxd', 'tests/*.py']}, + # include_package_data = True, + keywords=('functional utility itertools functools iterator generator ' + 'curry memoize lazy streaming bigdata cython toolz cytoolz'), + classifiers = [ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: BSD License', + 'Operating System :: OS Independent', + 'Programming Language :: Cython', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Information Analysis', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: Utilities', + ], + # zip_safe=False +) From 87611a0e9a31b42c2a83acdc8d90a3be04ea10b1 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 16:17:12 -0500 Subject: [PATCH 054/146] Exclude py34 + win64 --- .binstar.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.binstar.yml b/.binstar.yml index 257744c..bcc3940 100644 --- a/.binstar.yml +++ b/.binstar.yml @@ -15,3 +15,8 @@ script: build_targets: files: conda channels: main + +--- +platform: win-64 +engine: python=3.4 +exclude: true From 001c2c3d3627b4e8ae2a2ac262592bf547af6480 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 16:29:21 -0500 Subject: [PATCH 055/146] no libpython on win64 and python34 --- conda.recipe/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index b6869d3..56f7efd 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -11,11 +11,11 @@ requirements: - python - cython - toolz - - libpython # [win] + - libpython # [win and not (win64 and py34)] run: - python - - libpython # [win] + - libpython # [win and not (win64 and py34)] test: requires: From 4dd4f9211e44c170aa83d4fa2bba442fc3cc2359 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 21 Dec 2014 16:31:11 -0500 Subject: [PATCH 056/146] Put win64 py34 back in --- .binstar.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.binstar.yml b/.binstar.yml index bcc3940..257744c 100644 --- a/.binstar.yml +++ b/.binstar.yml @@ -15,8 +15,3 @@ script: build_targets: files: conda channels: main - ---- -platform: win-64 -engine: python=3.4 -exclude: true From 865330c4978038917cc29604e557b6802c393168 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 24 Feb 2015 11:26:43 -0600 Subject: [PATCH 057/146] Use absolute imports in pyx files to support Cython 0.17 and later. Updated required Cython version (>=0.17) in 'requirements_devel.txt' --- cytoolz/curried_exceptions.pyx | 2 +- cytoolz/dicttoolz.pyx | 2 +- cytoolz/functoolz.pyx | 4 ++-- cytoolz/itertoolz.pyx | 4 ++-- cytoolz/recipes.pyx | 4 ++-- requirements_devel.txt | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cytoolz/curried_exceptions.pyx b/cytoolz/curried_exceptions.pyx index 3332441..4f909c3 100644 --- a/cytoolz/curried_exceptions.pyx +++ b/cytoolz/curried_exceptions.pyx @@ -1,6 +1,6 @@ #cython: embedsignature=True from cpython.dict cimport PyDict_Check -from .dicttoolz cimport c_merge_with +from cytoolz.dicttoolz cimport c_merge_with __all__ = ['merge_with'] diff --git a/cytoolz/dicttoolz.pyx b/cytoolz/dicttoolz.pyx index d57ecd6..163a4f4 100644 --- a/cytoolz/dicttoolz.pyx +++ b/cytoolz/dicttoolz.pyx @@ -7,7 +7,7 @@ from cpython.list cimport PyList_Append, PyList_New from cpython.ref cimport PyObject # Locally defined bindings that differ from `cython.cpython` bindings -from .cpython cimport PtrObject_GetItem +from cytoolz.cpython cimport PtrObject_GetItem __all__ = ['merge', 'merge_with', 'valmap', 'keymap', 'itemmap', 'valfilter', diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 4663ac6..73878f1 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -1,7 +1,7 @@ #cython: embedsignature=True import inspect import sys -from .compatibility import filter as ifilter, map as imap, reduce +from cytoolz.compatibility import filter as ifilter, map as imap, reduce from cpython.dict cimport PyDict_Merge, PyDict_New from cpython.exc cimport PyErr_Clear, PyErr_ExceptionMatches, PyErr_Occurred @@ -13,7 +13,7 @@ from cpython.set cimport PyFrozenSet_New from cpython.tuple cimport PyTuple_Check, PyTuple_GET_SIZE # Locally defined bindings that differ from `cython.cpython` bindings -from .cpython cimport PtrObject_Call +from cytoolz.cpython cimport PtrObject_Call __all__ = ['identity', 'thread_first', 'thread_last', 'memoize', 'compose', diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index d9e1e32..a0173fc 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -9,13 +9,13 @@ from cpython.set cimport PySet_Add, PySet_Contains from cpython.tuple cimport PyTuple_GetSlice, PyTuple_New, PyTuple_SET_ITEM # Locally defined bindings that differ from `cython.cpython` bindings -from .cpython cimport PtrIter_Next, PtrObject_GetItem +from cytoolz.cpython cimport PtrIter_Next, PtrObject_GetItem from collections import deque from heapq import heapify, heappop, heapreplace from itertools import chain, islice from operator import itemgetter -from .compatibility import map, zip, zip_longest +from cytoolz.compatibility import map, zip, zip_longest __all__ = ['remove', 'accumulate', 'groupby', 'merge_sorted', 'interleave', diff --git a/cytoolz/recipes.pyx b/cytoolz/recipes.pyx index d0e06af..1a74b60 100644 --- a/cytoolz/recipes.pyx +++ b/cytoolz/recipes.pyx @@ -1,9 +1,9 @@ #cython: embedsignature=True from cpython.sequence cimport PySequence_Tuple -from .itertoolz cimport frequencies, pluck +from cytoolz.itertoolz cimport frequencies, pluck from itertools import groupby -from .compatibility import map +from cytoolz.compatibility import map __all__ = ['countby', 'partitionby'] diff --git a/requirements_devel.txt b/requirements_devel.txt index a1a97d9..c6a899d 100644 --- a/requirements_devel.txt +++ b/requirements_devel.txt @@ -1,3 +1,3 @@ -cython +cython>=0.17 nose -e git+https://github.com/pytoolz/toolz.git From 2784b8abdf8e1a180c5426c46945f45ca5a1e757 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 3 Mar 2015 21:05:20 -0600 Subject: [PATCH 058/146] Fix merge_sorted with key function (see #55). It was returning `key(item)` instead of `item` in a "shortcut" case. --- cytoolz/itertoolz.pyx | 3 +-- cytoolz/tests/test_itertoolz.py | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index a0173fc..aea350a 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -252,8 +252,7 @@ cdef class _merge_sorted_key: item = self.pq[0] self.shortcut = item[3] return item[2] - retval = next(self.shortcut) - return self.key(retval) + return next(self.shortcut) item = self.pq[0] retval = item[2] diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index a057606..62f46d6 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -75,6 +75,11 @@ def test_merge_sorted(): key=lambda x: -ord(x))) == 'cccbbbaaa' assert list(merge_sorted([1], [2, 3, 4], key=identity)) == [1, 2, 3, 4] + data = [[(1, 2), (0, 4), (3, 6)], [(5, 3), (6, 5), (8, 8)], + [(9, 1), (9, 8), (9, 9)]] + assert list(merge_sorted(*data, key=lambda x: x[1])) == [ + (9, 1), (1, 2), (5, 3), (0, 4), (6, 5), (3, 6), (8, 8), (9, 8), (9, 9)] + def test_interleave(): assert ''.join(interleave(('ABC', '123'))) == 'A1B2C3' From 4dfede735a4d1880087a5a53ae6c99111b4078d5 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 3 Mar 2015 21:50:50 -0600 Subject: [PATCH 059/146] Bump version to 0.7.3dev, and include *pyx files in distributed package. --- conda.recipe/meta.yaml | 2 +- cytoolz/_version.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 56f7efd..5c73925 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: cytoolz - version: "0.7.1" + version: "0.7.2" build: number: {{environ.get('BINSTAR_BUILD', 1)}} diff --git a/cytoolz/_version.py b/cytoolz/_version.py index f33da1a..7744946 100644 --- a/cytoolz/_version.py +++ b/cytoolz/_version.py @@ -1,2 +1,2 @@ -__version__ = '0.7.2dev' +__version__ = '0.7.3dev' __toolz_version__ = '0.7.1' diff --git a/setup.py b/setup.py index 6f5a1b5..9195f23 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,7 @@ maintainer_email='erik.n.welch@gmail.com', license = 'BSD', packages=['cytoolz'], - package_data={'cytoolz': ['*.pxd', 'tests/*.py']}, + package_data={'cytoolz': ['*.pyx', '*.pxd', 'tests/*.py']}, # include_package_data = True, keywords=('functional utility itertools functools iterator generator ' 'curry memoize lazy streaming bigdata cython toolz cytoolz'), From de43358e9f2293afcdbc81972688f1125e168b66 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 19 Mar 2015 23:18:38 -0500 Subject: [PATCH 060/146] Make running dev tests comparing toolz to cytoolz more strict. Also, cytoolz may be tested when toolz is not installed. Releases of cytoolz have no dependencies besides a C compiler. --- cytoolz/tests/dev_skip_test.py | 28 ++++++++++++++++++++++--- cytoolz/tests/test_curried_toolzlike.py | 7 +++++-- cytoolz/tests/test_docstrings.py | 2 +- cytoolz/tests/test_embedded_sigs.py | 2 +- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/cytoolz/tests/dev_skip_test.py b/cytoolz/tests/dev_skip_test.py index 3546f8a..03552f9 100644 --- a/cytoolz/tests/dev_skip_test.py +++ b/cytoolz/tests/dev_skip_test.py @@ -1,8 +1,30 @@ +""" +Determine when dev tests should be skipped by regular users. + +Some tests are only intended to be tested during development right +before performing a release. These do not test core functionality +of `cytoolz` and may be skipped. These tests are only run if the +following conditions are true: + + - toolz is installed + - toolz is the correct version + - cytoolz is a release version +""" import cytoolz from nose.tools import nottest, istest +try: + import toolz + do_toolz_tests = True +except ImportError: + do_toolz_tests = False + +if do_toolz_tests: + do_toolz_tests = cytoolz.__toolz_version__ == toolz.__version__ + do_toolz_tests &= 'dev' not in cytoolz.__version__ + # Decorator used to skip tests for developmental versions of CyToolz -if 'dev' in cytoolz.__version__: - dev_skip_test = nottest -else: +if do_toolz_tests: dev_skip_test = istest +else: + dev_skip_test = nottest diff --git a/cytoolz/tests/test_curried_toolzlike.py b/cytoolz/tests/test_curried_toolzlike.py index 2ebb99f..d5a4fa5 100644 --- a/cytoolz/tests/test_curried_toolzlike.py +++ b/cytoolz/tests/test_curried_toolzlike.py @@ -1,7 +1,5 @@ import cytoolz import cytoolz.curried -import toolz -import toolz.curried import types from dev_skip_test import dev_skip_test @@ -11,12 +9,15 @@ @dev_skip_test def test_toolzcurry_is_class(): + import toolz assert isinstance(toolz.curry, type) is True assert isinstance(toolz.curry, types.FunctionType) is False @dev_skip_test def test_cytoolz_like_toolz(): + import toolz + import toolz.curried for key, val in toolz.curried.__dict__.items(): if isinstance(val, toolz.curry): if val.func is toolz.curry: # XXX: Python 3.4 work-around! @@ -29,6 +30,8 @@ def test_cytoolz_like_toolz(): @dev_skip_test def test_toolz_like_cytoolz(): + import toolz + import toolz.curried for key, val in cytoolz.curried.__dict__.items(): if isinstance(val, cytoolz.curry): assert hasattr(toolz.curried, key), ( diff --git a/cytoolz/tests/test_docstrings.py b/cytoolz/tests/test_docstrings.py index 4be663c..d9711b1 100644 --- a/cytoolz/tests/test_docstrings.py +++ b/cytoolz/tests/test_docstrings.py @@ -1,6 +1,5 @@ import difflib import cytoolz -import toolz from cytoolz import curry, identity, keyfilter, valfilter, merge_with from cytoolz.utils import raises @@ -33,6 +32,7 @@ def convertdoc(doc): @dev_skip_test def test_docstrings_uptodate(): + import toolz differ = difflib.Differ() # only consider items created in both `toolz` and `cytoolz` diff --git a/cytoolz/tests/test_embedded_sigs.py b/cytoolz/tests/test_embedded_sigs.py index d2cd69f..1d037c2 100644 --- a/cytoolz/tests/test_embedded_sigs.py +++ b/cytoolz/tests/test_embedded_sigs.py @@ -1,6 +1,5 @@ import inspect import cytoolz -import toolz from types import BuiltinFunctionType from cytoolz import curry, identity, keyfilter, valfilter, merge_with @@ -18,6 +17,7 @@ def test_class_sigs(): """ Test that all ``cdef class`` extension types in ``cytoolz`` have correctly embedded the function signature as done in ``toolz``. """ + import toolz # only consider items created in both `toolz` and `cytoolz` toolz_dict = valfilter(isfrommod('toolz'), toolz.__dict__) cytoolz_dict = valfilter(isfrommod('cytoolz'), cytoolz.__dict__) From 9ce40e0f0d9ef849334c46452ddaa289f88a10ca Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 21 Mar 2015 12:13:56 -0500 Subject: [PATCH 061/146] Don't require nose to run tests (including dev tests). pytest (without nose installed) can run all tests, including dev tests. --- cytoolz/tests/dev_skip_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cytoolz/tests/dev_skip_test.py b/cytoolz/tests/dev_skip_test.py index 03552f9..82e4d91 100644 --- a/cytoolz/tests/dev_skip_test.py +++ b/cytoolz/tests/dev_skip_test.py @@ -11,7 +11,11 @@ - cytoolz is a release version """ import cytoolz -from nose.tools import nottest, istest +try: + from nose.tools import nottest, istest +except ImportError: + istest = lambda func: setattr(func, '__test__', True) or func + nottest = lambda func: setattr(func, '__test__', False) or func try: import toolz From 87f0f39b83c71991dc2a202ac4f61c1c11b89f96 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 21 Mar 2015 19:09:34 -0500 Subject: [PATCH 062/146] Add topk. --- cytoolz/__init__.pxd | 3 +- cytoolz/curried.py | 1 + cytoolz/functoolz.pyx | 4 ++ cytoolz/itertoolz.pxd | 3 ++ cytoolz/itertoolz.pyx | 82 ++++++++++++++++++++++++++++-- cytoolz/tests/test_functoolz.py | 89 ++++++++++++++++++++++++++++++++- cytoolz/tests/test_itertoolz.py | 21 +++++++- cytoolz/tests/test_none_safe.py | 4 ++ 8 files changed, 199 insertions(+), 8 deletions(-) diff --git a/cytoolz/__init__.pxd b/cytoolz/__init__.pxd index ba716bb..9207375 100644 --- a/cytoolz/__init__.pxd +++ b/cytoolz/__init__.pxd @@ -2,7 +2,8 @@ from cytoolz.itertoolz cimport ( accumulate, c_merge_sorted, cons, count, drop, get, groupby, first, frequencies, interleave, interpose, isdistinct, isiterable, iterate, last, mapcat, nth, partition, partition_all, pluck, reduceby, remove, - rest, second, sliding_window, take, tail, take_nth, unique, join) + rest, second, sliding_window, take, tail, take_nth, unique, join, + topk) from cytoolz.functoolz cimport ( diff --git a/cytoolz/curried.py b/cytoolz/curried.py index bc08af8..195e998 100644 --- a/cytoolz/curried.py +++ b/cytoolz/curried.py @@ -75,6 +75,7 @@ tail = cytoolz.curry(tail) take = cytoolz.curry(take) take_nth = cytoolz.curry(take_nth) +topk = cytoolz.curry(topk) unique = cytoolz.curry(unique) update_in = cytoolz.curry(update_in) valfilter = cytoolz.curry(valfilter) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 73878f1..883485d 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -252,6 +252,8 @@ cdef class curry: raise val def __get__(self, instance, owner): + if instance is None: + return self return curry(self, instance) def __reduce__(self): @@ -349,6 +351,8 @@ cdef class c_memoize: return result def __get__(self, instance, owner): + if instance is None: + return self return curry(self, instance) diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index a21e779..ad3d373 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -237,3 +237,6 @@ cdef class _outer_join_index(_outer_join): cdef class _outer_join_indices(_outer_join): pass + + +cpdef object topk(Py_ssize_t k, object seq, object key=*) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index aea350a..0c20eea 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -23,7 +23,7 @@ __all__ = ['remove', 'accumulate', 'groupby', 'merge_sorted', 'interleave', 'first', 'second', 'nth', 'last', 'get', 'concat', 'concatv', 'mapcat', 'cons', 'interpose', 'frequencies', 'reduceby', 'iterate', 'sliding_window', 'partition', 'partition_all', 'count', 'pluck', - 'join', 'tail'] + 'join', 'tail', 'topk'] concatv = chain @@ -113,11 +113,11 @@ cpdef dict groupby(object key, object seq): Group a collection by a key function >>> names = ['Alice', 'Bob', 'Charlie', 'Dan', 'Edith', 'Frank'] - >>> groupby(len, names) + >>> groupby(len, names) # doctest: +SKIP {3: ['Bob', 'Dan'], 5: ['Alice', 'Edith', 'Frank'], 7: ['Charlie']} >>> iseven = lambda x: x % 2 == 0 - >>> groupby(iseven, [1, 2, 3, 4, 5, 6, 7, 8]) + >>> groupby(iseven, [1, 2, 3, 4, 5, 6, 7, 8]) # doctest: +SKIP {False: [1, 3, 5, 7], True: [2, 4, 6, 8]} Non-callable keys imply grouping on a member. @@ -797,10 +797,10 @@ cpdef dict reduceby(object key, object binop, object seq, object init=no_default >>> data = [1, 2, 3, 4, 5] - >>> reduceby(iseven, add, data) + >>> reduceby(iseven, add, data) # doctest: +SKIP {False: 9, True: 6} - >>> reduceby(iseven, mul, data) + >>> reduceby(iseven, mul, data) # doctest: +SKIP {False: 15, True: 8} Complex Example @@ -1502,3 +1502,75 @@ cdef class _inner_join_indices(_inner_join): Py_INCREF(val) PyTuple_SET_ITEM(keyval, i, val) return keyval + + +cpdef object topk(Py_ssize_t k, object seq, object key=None): + """ + Find the k largest elements of a sequence + + Operates lazily in ``n*log(k)`` time + + >>> topk(2, [1, 100, 10, 1000]) + (1000, 100) + + Use a key function to change sorted order + + >>> topk(2, ['Alice', 'Bob', 'Charlie', 'Dan'], key=len) + ('Charlie', 'Alice') + + See also: + heapq.nlargest + """ + cdef object item, val, top + cdef object it = iter(seq) + cdef object _heapreplace = heapreplace + cdef Py_ssize_t i = k + cdef list pq = [] + + if key is not None and not callable(key): + key = getter(key) + + if k < 2: + if k < 1: + return () + top = list(take(1, it)) + if len(top) == 0: + return () + it = concatv(top, it) + if key is None: + return (max(it),) + else: + return (max(it, key=key),) + + for item in it: + if key is None: + PyList_Append(pq, (item, i)) + else: + PyList_Append(pq, (key(item), i, item)) + i -= 1 + if i == 0: + break + if i != 0: + pq.sort(reverse=True) + k = 0 if key is None else 2 + return tuple([item[k] for item in pq]) + + heapify(pq) + top = pq[0][0] + if key is None: + for item in it: + if top < item: + _heapreplace(pq, (item, i)) + top = pq[0][0] + i -= 1 + else: + for item in it: + val = key(item) + if top < val: + _heapreplace(pq, (val, i, item)) + top = pq[0][0] + i -= 1 + + pq.sort(reverse=True) + k = 0 if key is None else 2 + return tuple([item[k] for item in pq]) diff --git a/cytoolz/tests/test_functoolz.py b/cytoolz/tests/test_functoolz.py index 951c8c3..c15ba92 100644 --- a/cytoolz/tests/test_functoolz.py +++ b/cytoolz/tests/test_functoolz.py @@ -4,7 +4,7 @@ from operator import add, mul, itemgetter from cytoolz.utils import raises from functools import partial -from cytoolz.compatibility import reduce +from cytoolz.compatibility import reduce, PY3 def iseven(x): @@ -313,6 +313,93 @@ def f(x, y=0): assert cf(y=1)(y=2)(y=3)(1) == f(1, 3) +def test_curry_on_classmethods(): + class A(object): + BASE = 10 + + def __init__(self, base): + self.BASE = base + + @curry + def addmethod(self, x, y): + return self.BASE + x + y + + @classmethod + @curry + def addclass(cls, x, y): + return cls.BASE + x + y + + @staticmethod + @curry + def addstatic(x, y): + return x + y + + a = A(100) + assert a.addmethod(3, 4) == 107 + assert a.addmethod(3)(4) == 107 + assert A.addmethod(a, 3, 4) == 107 + assert A.addmethod(a)(3)(4) == 107 + + assert a.addclass(3, 4) == 17 + assert a.addclass(3)(4) == 17 + assert A.addclass(3, 4) == 17 + assert A.addclass(3)(4) == 17 + + assert a.addstatic(3, 4) == 7 + assert a.addstatic(3)(4) == 7 + assert A.addstatic(3, 4) == 7 + assert A.addstatic(3)(4) == 7 + + # we want this to be of type curry + assert isinstance(a.addmethod, curry) + assert isinstance(A.addmethod, curry) + + +def test_memoize_on_classmethods(): + class A(object): + BASE = 10 + HASH = 10 + + def __init__(self, base): + self.BASE = base + + @memoize + def addmethod(self, x, y): + return self.BASE + x + y + + @classmethod + @memoize + def addclass(cls, x, y): + return cls.BASE + x + y + + @staticmethod + @memoize + def addstatic(x, y): + return x + y + + def __hash__(self): + return self.HASH + + a = A(100) + assert a.addmethod(3, 4) == 107 + assert A.addmethod(a, 3, 4) == 107 + + a.BASE = 200 + assert a.addmethod(3, 4) == 107 + a.HASH = 200 + assert a.addmethod(3, 4) == 207 + + assert a.addclass(3, 4) == 17 + assert A.addclass(3, 4) == 17 + A.BASE = 20 + assert A.addclass(3, 4) == 17 + A.HASH = 20 # hashing of class is handled by metaclass + assert A.addclass(3, 4) == 17 # hence, != 27 + + assert a.addstatic(3, 4) == 7 + assert A.addstatic(3, 4) == 7 + + def test__num_required_args(): assert _num_required_args(map) != 0 assert _num_required_args(lambda x: x) == 1 diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index 62f46d6..6b0cb8f 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -10,7 +10,8 @@ rest, last, cons, frequencies, reduceby, iterate, accumulate, sliding_window, count, partition, - partition_all, take_nth, pluck, join) + partition_all, take_nth, pluck, join, + topk) from cytoolz.compatibility import range, filter from operator import add, mul @@ -416,3 +417,21 @@ def test_outer_join(): expected = set([(2, 2), (1, None), (None, 3)]) assert result == expected + + +def test_topk(): + assert topk(2, [4, 1, 5, 2]) == (5, 4) + assert topk(2, [4, 1, 5, 2], key=lambda x: -x) == (1, 2) + assert topk(2, iter([5, 1, 4, 2]), key=lambda x: -x) == (1, 2) + + assert topk(2, [{'a': 1, 'b': 10}, {'a': 2, 'b': 9}, + {'a': 10, 'b': 1}, {'a': 9, 'b': 2}], key='a') == \ + ({'a': 10, 'b': 1}, {'a': 9, 'b': 2}) + + assert topk(2, [{'a': 1, 'b': 10}, {'a': 2, 'b': 9}, + {'a': 10, 'b': 1}, {'a': 9, 'b': 2}], key='b') == \ + ({'a': 1, 'b': 10}, {'a': 2, 'b': 9}) + + +def test_topk_is_stable(): + assert topk(4, [5, 9, 2, 1, 5, 3], key=lambda x: 1) == (5, 9, 2, 1) diff --git a/cytoolz/tests/test_none_safe.py b/cytoolz/tests/test_none_safe.py index a76944f..6e4d6a4 100644 --- a/cytoolz/tests/test_none_safe.py +++ b/cytoolz/tests/test_none_safe.py @@ -278,6 +278,10 @@ def test_itertoolz(): assert raises(TypeError, lambda: join(first, (1, 2, 3), second, None)) tested.append('join') + assert raises(TypeError, lambda: topk(None, [1, 2, 3])) + assert raises(TypeError, lambda: topk(3, None)) + tested.append('topk') + s1 = set(tested) s2 = set(cytoolz.itertoolz.__all__) assert s1 == s2, '%s not tested for being None-safe' % ', '.join(s2 - s1) From 72b9c36cfc2179e7ac2c50f105ef51888a346729 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 23 Apr 2015 00:15:55 -0500 Subject: [PATCH 063/146] This is one way to fix the memory leak in #63. I actually don't know how, where, or why the reference count is being increased by one, which requires us to decrement it. As such, I'm not entirely comfortable with the current solution or level of understanding. I would like to investigate further. --- cytoolz/dicttoolz.pyx | 3 ++- cytoolz/itertoolz.pyx | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cytoolz/dicttoolz.pyx b/cytoolz/dicttoolz.pyx index 163a4f4..7f339c7 100644 --- a/cytoolz/dicttoolz.pyx +++ b/cytoolz/dicttoolz.pyx @@ -4,7 +4,7 @@ from cpython.dict cimport (PyDict_Check, PyDict_GetItem, PyDict_Merge, PyDict_Update, PyDict_DelItem) from cpython.exc cimport PyErr_Clear, PyErr_GivenExceptionMatches, PyErr_Occurred from cpython.list cimport PyList_Append, PyList_New -from cpython.ref cimport PyObject +from cpython.ref cimport PyObject, Py_XDECREF # Locally defined bindings that differ from `cython.cpython` bindings from cytoolz.cpython cimport PtrObject_GetItem @@ -407,5 +407,6 @@ cpdef object get_in(object keys, object coll, object default=None, object no_def raise item PyErr_Clear() return default + Py_XDECREF(obj) coll = obj return coll diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 0c20eea..af10373 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -661,7 +661,6 @@ cpdef object get(object ind, object seq, object default=no_default): PyTuple_SET_ITEM(result, i, default) else: val = obj - Py_INCREF(val) PyTuple_SET_ITEM(result, i, val) return result @@ -674,6 +673,7 @@ cpdef object get(object ind, object seq, object default=no_default): PyErr_Clear() return default raise val + Py_XDECREF(obj) return obj @@ -1060,6 +1060,7 @@ cdef class _pluck_index_default: raise PyErr_Occurred() PyErr_Clear() return self.default + Py_XDECREF(obj) return obj @@ -1111,8 +1112,7 @@ cdef class _pluck_list_default: PyTuple_SET_ITEM(result, i, self.default) else: val = obj - Py_INCREF(val) - PyTuple_SET_ITEM(result, i, val) # TODO: redefine with "PyObject* val" and avoid cast + PyTuple_SET_ITEM(result, i, val) return result From 9033ed35ebf8b2672066f16400f06513553e7df8 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 25 Apr 2015 10:22:03 -0500 Subject: [PATCH 064/146] Fix memory leak in curry. Lesson learned: binding a C function to return type then casting that to type increments the reference count of the object one more than if the function were bound to return type . All memory leaks due to my misunderstanding of this have been fixed. --- cytoolz/functoolz.pyx | 3 ++- cytoolz/itertoolz.pyx | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 883485d..95ab620 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -7,7 +7,7 @@ from cpython.dict cimport PyDict_Merge, PyDict_New from cpython.exc cimport PyErr_Clear, PyErr_ExceptionMatches, PyErr_Occurred from cpython.object cimport (PyCallable_Check, PyObject_Call, PyObject_CallObject, PyObject_RichCompare, Py_EQ, Py_NE) -from cpython.ref cimport PyObject +from cpython.ref cimport PyObject, Py_DECREF from cpython.sequence cimport PySequence_Concat from cpython.set cimport PyFrozenSet_New from cpython.tuple cimport PyTuple_Check, PyTuple_GET_SIZE @@ -240,6 +240,7 @@ cdef class curry: obj = PtrObject_Call(self.func, args, kwargs) if obj is not NULL: val = obj + Py_DECREF(val) return val val = PyErr_Occurred() diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index af10373..c558970 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -3,7 +3,7 @@ from cpython.dict cimport PyDict_GetItem, PyDict_SetItem from cpython.exc cimport (PyErr_Clear, PyErr_ExceptionMatches, PyErr_GivenExceptionMatches, PyErr_Occurred) from cpython.list cimport (PyList_Append, PyList_GET_ITEM, PyList_GET_SIZE) -from cpython.ref cimport PyObject, Py_DECREF, Py_INCREF, Py_XDECREF +from cpython.ref cimport PyObject, Py_INCREF, Py_XDECREF from cpython.sequence cimport PySequence_Check from cpython.set cimport PySet_Add, PySet_Contains from cpython.tuple cimport PyTuple_GetSlice, PyTuple_New, PyTuple_SET_ITEM From 19d5278654ae911bc9cc5ce19be1b88694fda77b Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 17 May 2015 12:26:20 -0500 Subject: [PATCH 065/146] Add diff and prepare for "factory" keyword argument in dictoolz. Goal is to have a release to match toolz 0.7.2 --- cytoolz/__init__.pxd | 2 +- cytoolz/_version.py | 4 +- cytoolz/curried.py | 1 + cytoolz/curried_exceptions.pyx | 16 +- cytoolz/dicttoolz.pxd | 16 +- cytoolz/dicttoolz.pyx | 20 +- cytoolz/functoolz.pyx | 7 +- cytoolz/itertoolz.pxd | 14 ++ cytoolz/itertoolz.pyx | 88 +++++++- cytoolz/tests/test_compatibility.py | 18 ++ cytoolz/tests/test_curried.py | 10 +- cytoolz/tests/test_dicttoolz.py | 315 +++++++++++++++++++--------- cytoolz/tests/test_functoolz.py | 14 ++ cytoolz/tests/test_itertoolz.py | 41 +++- cytoolz/tests/test_none_safe.py | 6 + cytoolz/tests/test_recipes.py | 2 - 16 files changed, 439 insertions(+), 135 deletions(-) create mode 100644 cytoolz/tests/test_compatibility.py diff --git a/cytoolz/__init__.pxd b/cytoolz/__init__.pxd index 9207375..af7bf39 100644 --- a/cytoolz/__init__.pxd +++ b/cytoolz/__init__.pxd @@ -3,7 +3,7 @@ from cytoolz.itertoolz cimport ( frequencies, interleave, interpose, isdistinct, isiterable, iterate, last, mapcat, nth, partition, partition_all, pluck, reduceby, remove, rest, second, sliding_window, take, tail, take_nth, unique, join, - topk) + c_diff, topk) from cytoolz.functoolz cimport ( diff --git a/cytoolz/_version.py b/cytoolz/_version.py index 7744946..a86b93a 100644 --- a/cytoolz/_version.py +++ b/cytoolz/_version.py @@ -1,2 +1,2 @@ -__version__ = '0.7.3dev' -__toolz_version__ = '0.7.1' +__version__ = '0.7.3' +__toolz_version__ = '0.7.2' diff --git a/cytoolz/curried.py b/cytoolz/curried.py index 195e998..722b05b 100644 --- a/cytoolz/curried.py +++ b/cytoolz/curried.py @@ -61,6 +61,7 @@ map = cytoolz.curry(map) mapcat = cytoolz.curry(mapcat) memoize = cytoolz.curry(memoize) +merge = cytoolz.curry(merge) merge_with = cytoolz.curry(merge_with) nth = cytoolz.curry(nth) partition = cytoolz.curry(partition) diff --git a/cytoolz/curried_exceptions.pyx b/cytoolz/curried_exceptions.pyx index 4f909c3..3a9e98f 100644 --- a/cytoolz/curried_exceptions.pyx +++ b/cytoolz/curried_exceptions.pyx @@ -1,11 +1,19 @@ #cython: embedsignature=True from cpython.dict cimport PyDict_Check -from cytoolz.dicttoolz cimport c_merge_with +from cytoolz.dicttoolz cimport c_merge, c_merge_with -__all__ = ['merge_with'] +__all__ = ['merge', 'merge_with'] -def merge_with(func, *dicts): +def merge(*dicts, **kwargs): + if len(dicts) == 0: + raise TypeError() + if len(dicts) == 1 and not PyDict_Check(dicts[0]): + dicts = dicts[0] + return c_merge(dicts) + + +def merge_with(func, *dicts, **kwargs): """ Merge dictionaries and apply function to combined values @@ -22,7 +30,7 @@ def merge_with(func, *dicts): merge """ if len(dicts) == 0: - raise TypeError + raise TypeError() if len(dicts) == 1 and not PyDict_Check(dicts[0]): dicts = dicts[0] diff --git a/cytoolz/dicttoolz.pxd b/cytoolz/dicttoolz.pxd index bf798f9..c01b4bf 100644 --- a/cytoolz/dicttoolz.pxd +++ b/cytoolz/dicttoolz.pxd @@ -4,31 +4,31 @@ cdef dict c_merge(object dicts) cdef dict c_merge_with(object func, object dicts) -cpdef dict valmap(object func, dict d) +cpdef dict valmap(object func, dict d, object factory=*) -cpdef dict keymap(object func, dict d) +cpdef dict keymap(object func, dict d, object factory=*) -cpdef dict itemmap(object func, dict d) +cpdef dict itemmap(object func, dict d, object factory=*) -cpdef dict valfilter(object predicate, dict d) +cpdef dict valfilter(object predicate, dict d, object factory=*) -cpdef dict keyfilter(object predicate, dict d) +cpdef dict keyfilter(object predicate, dict d, object factory=*) -cpdef dict itemfilter(object predicate, dict d) +cpdef dict itemfilter(object predicate, dict d, object factory=*) -cpdef dict assoc(dict d, object key, object value) +cpdef dict assoc(dict d, object key, object value, object factory=*) cpdef dict dissoc(dict d, object key) -cpdef dict update_in(dict d, object keys, object func, object default=*) +cpdef dict update_in(dict d, object keys, object func, object default=*, object factory=*) cpdef object get_in(object keys, object coll, object default=*, object no_default=*) diff --git a/cytoolz/dicttoolz.pyx b/cytoolz/dicttoolz.pyx index 7f339c7..5b0303e 100644 --- a/cytoolz/dicttoolz.pyx +++ b/cytoolz/dicttoolz.pyx @@ -22,7 +22,7 @@ cdef dict c_merge(object dicts): return rv -def merge(*dicts): +def merge(*dicts, **kwargs): """ Merge a collection of dictionaries @@ -63,7 +63,7 @@ cdef dict c_merge_with(object func, object dicts): return rv -def merge_with(func, *dicts): +def merge_with(func, *dicts, **kwargs): """ Merge dictionaries and apply function to combined values @@ -84,7 +84,7 @@ def merge_with(func, *dicts): return c_merge_with(func, dicts) -cpdef dict valmap(object func, dict d): +cpdef dict valmap(object func, dict d, object factory=dict): """ Apply function to values of dictionary @@ -112,7 +112,7 @@ cpdef dict valmap(object func, dict d): return rv -cpdef dict keymap(object func, dict d): +cpdef dict keymap(object func, dict d, object factory=dict): """ Apply function to keys of dictionary @@ -140,7 +140,7 @@ cpdef dict keymap(object func, dict d): return rv -cpdef dict itemmap(object func, dict d): +cpdef dict itemmap(object func, dict d, object factory=dict): """ Apply function to items of dictionary @@ -170,7 +170,7 @@ cpdef dict itemmap(object func, dict d): return rv -cpdef dict valfilter(object predicate, dict d): +cpdef dict valfilter(object predicate, dict d, object factory=dict): """ Filter items in dictionary by value @@ -201,7 +201,7 @@ cpdef dict valfilter(object predicate, dict d): return rv -cpdef dict keyfilter(object predicate, dict d): +cpdef dict keyfilter(object predicate, dict d, object factory=dict): """ Filter items in dictionary by key @@ -232,7 +232,7 @@ cpdef dict keyfilter(object predicate, dict d): return rv -cpdef dict itemfilter(object predicate, dict d): +cpdef dict itemfilter(object predicate, dict d, object factory=dict): """ Filter items in dictionary by item @@ -266,7 +266,7 @@ cpdef dict itemfilter(object predicate, dict d): return rv -cpdef dict assoc(dict d, object key, object value): +cpdef dict assoc(dict d, object key, object value, object factory=dict): """ Return a new dict with new key value pair @@ -299,7 +299,7 @@ cpdef dict dissoc(dict d, object key): return rv -cpdef dict update_in(dict d, object keys, object func, object default=None): +cpdef dict update_in(dict d, object keys, object func, object default=None, object factory=dict): """ Update value in a (potentially) nested dictionary diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 95ab620..c4969e6 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -152,7 +152,7 @@ cpdef Py_ssize_t _num_required_args(object func) except *: cdef class curry: - """ curry(self, func, *args, **kwargs) + """ curry(self, *args, **kwargs) Curry a callable function @@ -181,7 +181,10 @@ cdef class curry: cytoolz.curried - namespace of curried functions http://toolz.readthedocs.org/en/latest/curry.html """ - def __cinit__(self, func, *args, **kwargs): + def __cinit__(self, *args, **kwargs): + if not args: + raise TypeError('__init__() takes at least 2 arguments (1 given)') + func, args = args[0], args[1:] if not PyCallable_Check(func): raise TypeError("Input must be callable") diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index ad3d373..8b98424 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -239,4 +239,18 @@ cdef class _outer_join_indices(_outer_join): pass +cdef class _diff_key: + cdef Py_ssize_t N + cdef object iters + cdef object key + + +cdef class _diff_identity: + cdef Py_ssize_t N + cdef object iters + + +cdef object c_diff(object seqs, object default=*, object key=*) + + cpdef object topk(Py_ssize_t k, object seq, object key=*) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index c558970..cbb8c3d 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -3,10 +3,11 @@ from cpython.dict cimport PyDict_GetItem, PyDict_SetItem from cpython.exc cimport (PyErr_Clear, PyErr_ExceptionMatches, PyErr_GivenExceptionMatches, PyErr_Occurred) from cpython.list cimport (PyList_Append, PyList_GET_ITEM, PyList_GET_SIZE) +from cpython.object cimport PyObject_RichCompareBool, Py_NE from cpython.ref cimport PyObject, Py_INCREF, Py_XDECREF from cpython.sequence cimport PySequence_Check from cpython.set cimport PySet_Add, PySet_Contains -from cpython.tuple cimport PyTuple_GetSlice, PyTuple_New, PyTuple_SET_ITEM +from cpython.tuple cimport PyTuple_GET_ITEM, PyTuple_GetSlice, PyTuple_New, PyTuple_SET_ITEM # Locally defined bindings that differ from `cython.cpython` bindings from cytoolz.cpython cimport PtrIter_Next, PtrObject_GetItem @@ -23,7 +24,7 @@ __all__ = ['remove', 'accumulate', 'groupby', 'merge_sorted', 'interleave', 'first', 'second', 'nth', 'last', 'get', 'concat', 'concatv', 'mapcat', 'cons', 'interpose', 'frequencies', 'reduceby', 'iterate', 'sliding_window', 'partition', 'partition_all', 'count', 'pluck', - 'join', 'tail', 'topk'] + 'join', 'tail', 'diff', 'topk'] concatv = chain @@ -1504,6 +1505,89 @@ cdef class _inner_join_indices(_inner_join): return keyval +cdef class _diff_key: + def __cinit__(self, object seqs, object key, object default=no_default): + self.N = len(seqs) + if self.N < 2: + raise TypeError('Too few sequences given (min 2 required)') + if default == no_default: + self.iters = zip(*seqs) + else: + self.iters = zip_longest(*seqs, fillvalue=default) + self.key = key + + def __iter__(self): + return self + + def __next__(self): + cdef object val, val2, items + cdef Py_ssize_t i + while True: + items = next(self.iters) + val = self.key(PyTuple_GET_ITEM(items, 0)) + for i in range(1, self.N): + val2 = self.key(PyTuple_GET_ITEM(items, i)) + if PyObject_RichCompareBool(val, val2, Py_NE): + return items + +cdef class _diff_identity: + def __cinit__(self, object seqs, object default=no_default): + self.N = len(seqs) + if self.N < 2: + raise TypeError('Too few sequences given (min 2 required)') + if default == no_default: + self.iters = zip(*seqs) + else: + self.iters = zip_longest(*seqs, fillvalue=default) + + def __iter__(self): + return self + + def __next__(self): + cdef object val, val2, items + cdef Py_ssize_t i + while True: + items = next(self.iters) + val = PyTuple_GET_ITEM(items, 0) + for i in range(1, self.N): + val2 = PyTuple_GET_ITEM(items, i) + if PyObject_RichCompareBool(val, val2, Py_NE): + return items + + +cdef object c_diff(object seqs, object default=no_default, object key=None): + if key is None: + return _diff_identity(seqs, default=default) + else: + return _diff_key(seqs, key, default=default) + + +def diff(*seqs, **kwargs): + """ + Return those items that differ between sequences + + >>> list(diff([1, 2, 3], [1, 2, 10, 100])) + [(3, 10)] + + Shorter sequences may be padded with a ``default`` value: + + >>> list(diff([1, 2, 3], [1, 2, 10, 100], default=None)) + [(3, 10), (None, 100)] + + A ``key`` function may also be applied to each item to use during + comparisons: + + >>> list(diff(['apples', 'bananas'], ['Apples', 'Oranges'], key=str.lower)) + [('bananas', 'Oranges')] + """ + N = len(seqs) + if N == 1 and isinstance(seqs[0], list): + seqs = seqs[0] + default = kwargs.get('default', no_default) + key = kwargs.get('key') + return c_diff(seqs, default=default, key=key) + + cpdef object topk(Py_ssize_t k, object seq, object key=None): """ Find the k largest elements of a sequence diff --git a/cytoolz/tests/test_compatibility.py b/cytoolz/tests/test_compatibility.py new file mode 100644 index 0000000..fafd65e --- /dev/null +++ b/cytoolz/tests/test_compatibility.py @@ -0,0 +1,18 @@ +from cytoolz.compatibility import map, filter, iteritems, iterkeys, itervalues + + +def test_map_filter_are_lazy(): + def bad(x): + raise Exception() + map(bad, [1, 2, 3]) + filter(bad, [1, 2, 3]) + + +def test_dict_iteration(): + d = {'a': 1, 'b': 2, 'c': 3} + assert not isinstance(iteritems(d), list) + assert not isinstance(iterkeys(d), list) + assert not isinstance(itervalues(d), list) + assert set(iteritems(d)) == set(d.items()) + assert set(iterkeys(d)) == set(d.keys()) + assert set(itervalues(d)) == set(d.values()) diff --git a/cytoolz/tests/test_curried.py b/cytoolz/tests/test_curried.py index 4506ce9..6bd424d 100644 --- a/cytoolz/tests/test_curried.py +++ b/cytoolz/tests/test_curried.py @@ -1,6 +1,8 @@ import cytoolz import cytoolz.curried -from cytoolz.curried import take, first, second, sorted, merge_with, reduce +from cytoolz.curried import (take, first, second, sorted, merge_with, reduce, + merge) +from collections import defaultdict from operator import add @@ -12,6 +14,12 @@ def test_first(): assert first is cytoolz.itertoolz.first +def test_merge(): + assert merge(factory=lambda: defaultdict(int))({1: 1}) == {1: 1} + assert merge({1: 1}) == {1: 1} + assert merge({1: 1}, factory=lambda: defaultdict(int)) == {1: 1} + + def test_merge_with(): assert merge_with(sum)({1: 1}, {1: 2}) == {1: 3} diff --git a/cytoolz/tests/test_dicttoolz.py b/cytoolz/tests/test_dicttoolz.py index e607c26..7d07f60 100644 --- a/cytoolz/tests/test_dicttoolz.py +++ b/cytoolz/tests/test_dicttoolz.py @@ -1,110 +1,237 @@ -from cytoolz.utils import raises +from collections import defaultdict as _defaultdict from cytoolz.dicttoolz import (merge, merge_with, valmap, keymap, update_in, assoc, dissoc, keyfilter, valfilter, itemmap, itemfilter) +from cytoolz.utils import raises +from cytoolz.compatibility import PY3 + + +def inc(x): + return x + 1 + + +def iseven(i): + return i % 2 == 0 + + +class TestDict(object): + """Test typical usage: dict inputs, no factory keyword. + + Class attributes: + D: callable that inputs a dict and creates or returns a MutableMapping + kw: kwargs dict to specify "factory" keyword (if applicable) + """ + D = dict + kw = {} + + def test_merge(self): + D, kw = self.D, self.kw + assert merge(D({1: 1, 2: 2}), D({3: 4}), **kw) == D({1: 1, 2: 2, 3: 4}) + + def test_merge_iterable_arg(self): + D, kw = self.D, self.kw + assert merge([D({1: 1, 2: 2}), D({3: 4})], **kw) == D({1: 1, 2: 2, 3: 4}) + + def test_merge_with(self): + D, kw = self.D, self.kw + dicts = D({1: 1, 2: 2}), D({1: 10, 2: 20}) + assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22}) + assert merge_with(tuple, *dicts, **kw) == D({1: (1, 10), 2: (2, 20)}) + + dicts = D({1: 1, 2: 2, 3: 3}), D({1: 10, 2: 20}) + assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22, 3: 3}) + assert merge_with(tuple, *dicts, **kw) == D({1: (1, 10), 2: (2, 20), 3: (3,)}) + + assert not merge_with(sum) + + def test_merge_with_iterable_arg(self): + D, kw = self.D, self.kw + dicts = D({1: 1, 2: 2}), D({1: 10, 2: 20}) + assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22}) + assert merge_with(sum, dicts, **kw) == D({1: 11, 2: 22}) + assert merge_with(sum, iter(dicts), **kw) == D({1: 11, 2: 22}) + + def test_valmap(self): + D, kw = self.D, self.kw + assert valmap(inc, D({1: 1, 2: 2}), **kw) == D({1: 2, 2: 3}) + + def test_keymap(self): + D, kw = self.D, self.kw + assert keymap(inc, D({1: 1, 2: 2}), **kw) == D({2: 1, 3: 2}) + + def test_itemmap(self): + D, kw = self.D, self.kw + assert itemmap(reversed, D({1: 2, 2: 4}), **kw) == D({2: 1, 4: 2}) + + def test_valfilter(self): + D, kw = self.D, self.kw + assert valfilter(iseven, D({1: 2, 2: 3}), **kw) == D({1: 2}) + + def test_keyfilter(self): + D, kw = self.D, self.kw + assert keyfilter(iseven, D({1: 2, 2: 3}), **kw) == D({2: 3}) + + def test_itemfilter(self): + D, kw = self.D, self.kw + assert itemfilter(lambda item: iseven(item[0]), D({1: 2, 2: 3}), **kw) == D({2: 3}) + assert itemfilter(lambda item: iseven(item[1]), D({1: 2, 2: 3}), **kw) == D({1: 2}) + + def test_assoc(self): + D, kw = self.D, self.kw + assert assoc(D({}), "a", 1, **kw) == D({"a": 1}) + assert assoc(D({"a": 1}), "a", 3, **kw) == D({"a": 3}) + assert assoc(D({"a": 1}), "b", 3, **kw) == D({"a": 1, "b": 3}) + + # Verify immutability: + d = D({'x': 1}) + oldd = d + assoc(d, 'x', 2, **kw) + assert d is oldd + + def test_dissoc(self): + D, kw = self.D, self.kw + assert dissoc(D({"a": 1}), "a") == D({}) + assert dissoc(D({"a": 1, "b": 2}), "a") == D({"b": 2}) + assert dissoc(D({"a": 1, "b": 2}), "b") == D({"a": 1}) + + # Verify immutability: + d = D({'x': 1}) + oldd = d + d2 = dissoc(d, 'x') + assert d is oldd + assert d2 is not oldd + + def test_update_in(self): + D, kw = self.D, self.kw + assert update_in(D({"a": 0}), ["a"], inc, **kw) == D({"a": 1}) + assert update_in(D({"a": 0, "b": 1}), ["b"], str, **kw) == D({"a": 0, "b": "1"}) + assert (update_in(D({"t": 1, "v": D({"a": 0})}), ["v", "a"], inc, **kw) == + D({"t": 1, "v": D({"a": 1})})) + # Handle one missing key. + assert update_in(D({}), ["z"], str, None, **kw) == D({"z": "None"}) + assert update_in(D({}), ["z"], inc, 0, **kw) == D({"z": 1}) + assert update_in(D({}), ["z"], lambda x: x+"ar", default="b", **kw) == D({"z": "bar"}) + # Same semantics as Clojure for multiple missing keys, ie. recursively + # create nested empty dictionaries to the depth specified by the + # keys with the innermost value set to f(default). + assert update_in(D({}), [0, 1], inc, default=-1, **kw) == D({0: D({1: 0})}) + assert update_in(D({}), [0, 1], str, default=100, **kw) == D({0: D({1: "100"})}) + assert (update_in(D({"foo": "bar", 1: 50}), ["d", 1, 0], str, 20, **kw) == + D({"foo": "bar", 1: 50, "d": D({1: D({0: "20"})})})) + # Verify immutability: + d = D({'x': 1}) + oldd = d + update_in(d, ['x'], inc, **kw) + assert d is oldd + + def test_factory(self): + D, kw = self.D, self.kw + assert merge(defaultdict(int, D({1: 2})), D({2: 3})) == {1: 2, 2: 3} + assert (merge(defaultdict(int, D({1: 2})), D({2: 3}), + factory=lambda: defaultdict(int)) == + defaultdict(int, D({1: 2, 2: 3}))) + assert not (merge(defaultdict(int, D({1: 2})), D({2: 3}), + factory=lambda: defaultdict(int)) == {1: 2, 2: 3}) + assert raises(TypeError, lambda: merge(D({1: 2}), D({2: 3}), factoryy=dict)) + + +class defaultdict(_defaultdict): + def __eq__(self, other): + return (super(defaultdict, self).__eq__(other) and + isinstance(other, _defaultdict) and + self.default_factory == other.default_factory) + + +class TestDefaultDict(TestDict): + """Test defaultdict as input and factory + + Class attributes: + D: callable that inputs a dict and creates or returns a MutableMapping + kw: kwargs dict to specify "factory" keyword (if applicable) + """ + @staticmethod + def D(dict_): + return defaultdict(int, dict_) + + kw = {'factory': lambda: defaultdict(int)} + + +class CustomMapping(object): + """Define methods of the MutableMapping protocol required by dicttoolz""" + def __init__(self, *args, **kwargs): + self._d = dict(*args, **kwargs) + + def __getitem__(self, key): + return self._d[key] + + def __setitem__(self, key, val): + self._d[key] = val + + def __delitem__(self, key): + del self._d[key] + + def __iter__(self): + return iter(self._d) + + def __len__(self): + return len(self._d) + + def __contains__(self, key): + return key in self._d + + def __eq__(self, other): + return isinstance(other, CustomMapping) and self._d == other._d + + def __ne__(self, other): + return not isinstance(other, CustomMapping) or self._d != other._d + + def keys(self): + return self._d.keys() + + def values(self): + return self._d.values() + + def items(self): + return self._d.items() + def update(self, *args, **kwargs): + self._d.update(*args, **kwargs) -inc = lambda x: x + 1 - - -iseven = lambda i: i % 2 == 0 - - -def test_merge(): - assert merge({1: 1, 2: 2}, {3: 4}) == {1: 1, 2: 2, 3: 4} - - -def test_merge_iterable_arg(): - assert merge([{1: 1, 2: 2}, {3: 4}]) == {1: 1, 2: 2, 3: 4} - - -def test_merge_with(): - dicts = {1: 1, 2: 2}, {1: 10, 2: 20} - assert merge_with(sum, *dicts) == {1: 11, 2: 22} - assert merge_with(tuple, *dicts) == {1: (1, 10), 2: (2, 20)} - - dicts = {1: 1, 2: 2, 3: 3}, {1: 10, 2: 20} - assert merge_with(sum, *dicts) == {1: 11, 2: 22, 3: 3} - assert merge_with(tuple, *dicts) == {1: (1, 10), 2: (2, 20), 3: (3,)} - - assert not merge_with(sum) - - -def test_merge_with_iterable_arg(): - dicts = {1: 1, 2: 2}, {1: 10, 2: 20} - assert merge_with(sum, *dicts) == {1: 11, 2: 22} - assert merge_with(sum, dicts) == {1: 11, 2: 22} - assert merge_with(sum, iter(dicts)) == {1: 11, 2: 22} - - -def test_valmap(): - assert valmap(inc, {1: 1, 2: 2}) == {1: 2, 2: 3} - - -def test_keymap(): - assert keymap(inc, {1: 1, 2: 2}) == {2: 1, 3: 2} - - -def test_itemmap(): - assert itemmap(reversed, {1: 2, 2: 4}) == {2: 1, 4: 2} - - -def test_valfilter(): - assert valfilter(iseven, {1: 2, 2: 3}) == {1: 2} - + # Should we require these to be defined for Python 2? + if not PY3: + def iterkeys(self): + return self._d.iterkeys() -def test_keyfilter(): - assert keyfilter(iseven, {1: 2, 2: 3}) == {2: 3} + def itervalues(self): + return self._d.itervalues() + def iteritems(self): + return self._d.iteritems() -def test_itemfilter(): - assert itemfilter(lambda item: iseven(item[0]), {1: 2, 2: 3}) == {2: 3} - assert itemfilter(lambda item: iseven(item[1]), {1: 2, 2: 3}) == {1: 2} + # Unused methods that are part of the MutableMapping protocol + #def get(self, key, *args): + # return self._d.get(key, *args) + #def pop(self, key, *args): + # return self._d.pop(key, *args) -def test_assoc(): - assert assoc({}, "a", 1) == {"a": 1} - assert assoc({"a": 1}, "a", 3) == {"a": 3} - assert assoc({"a": 1}, "b", 3) == {"a": 1, "b": 3} + #def popitem(self, key): + # return self._d.popitem() - # Verify immutability: - d = {'x': 1} - oldd = d - assoc(d, 'x', 2) - assert d is oldd + #def clear(self): + # self._d.clear() + #def setdefault(self, key, *args): + # return self._d.setdefault(self, key, *args) -def test_dissoc(): - assert dissoc({"a": 1}, "a") == {} - assert dissoc({"a": 1, "b": 2}, "a") == {"b": 2} - assert dissoc({"a": 1, "b": 2}, "b") == {"a": 1} - # Verify immutability: - d = {'x': 1} - oldd = d - d2 = dissoc(d, 'x') - assert d is oldd - assert d2 is not oldd +class TestCustomMapping(TestDict): + """Test CustomMapping as input and factory + Class attributes: + D: callable that inputs a dict and creates or returns a MutableMapping + kw: kwargs dict to specify "factory" keyword (if applicable) + """ + D = CustomMapping + kw = {'factory': lambda: CustomMapping()} -def test_update_in(): - assert update_in({"a": 0}, ["a"], inc) == {"a": 1} - assert update_in({"a": 0, "b": 1}, ["b"], str) == {"a": 0, "b": "1"} - assert (update_in({"t": 1, "v": {"a": 0}}, ["v", "a"], inc) == - {"t": 1, "v": {"a": 1}}) - # Handle one missing key. - assert update_in({}, ["z"], str, None) == {"z": "None"} - assert update_in({}, ["z"], inc, 0) == {"z": 1} - assert update_in({}, ["z"], lambda x: x+"ar", default="b") == {"z": "bar"} - # Same semantics as Clojure for multiple missing keys, ie. recursively - # create nested empty dictionaries to the depth specified by the - # keys with the innermost value set to f(default). - assert update_in({}, [0, 1], inc, default=-1) == {0: {1: 0}} - assert update_in({}, [0, 1], str, default=100) == {0: {1: "100"}} - assert (update_in({"foo": "bar", 1: 50}, ["d", 1, 0], str, 20) == - {"foo": "bar", 1: 50, "d": {1: {0: "20"}}}) - # Verify immutability: - d = {'x': 1} - oldd = d - update_in(d, ['x'], inc) - assert d is oldd diff --git a/cytoolz/tests/test_functoolz.py b/cytoolz/tests/test_functoolz.py index c15ba92..acd8d37 100644 --- a/cytoolz/tests/test_functoolz.py +++ b/cytoolz/tests/test_functoolz.py @@ -1,3 +1,6 @@ +import platform + + from cytoolz.functoolz import (thread_first, thread_last, memoize, curry, compose, pipe, complement, do, juxt) from cytoolz.functoolz import _num_required_args @@ -161,6 +164,7 @@ def test_curry_simple(): cmap = curry(map) assert list(cmap(inc)([1, 2, 3])) == [2, 3, 4] + assert raises(TypeError, lambda: curry()) assert raises(TypeError, lambda: curry({1: 2})) @@ -186,6 +190,16 @@ def g(a=1, b=10, c=0): assert cg(0) == 2 # pass "a" as arg, not kwarg assert raises(TypeError, lambda: cg(1, 2)) # pass "b" as arg AND kwarg + def h(x, func=int): + return func(x) + + if platform.python_implementation() != 'PyPy'\ + or platform.python_version_tuple()[0] != '3': # Bug on PyPy3<2.5 + # __init__ must not pick func as positional arg + assert curry(h)(0.0) == 0 + assert curry(h)(func=str)(0.0) == '0.0' + assert curry(h, func=str)(0.0) == '0.0' + def test_curry_passes_errors(): @curry diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index 6b0cb8f..fcdd53b 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -11,7 +11,7 @@ reduceby, iterate, accumulate, sliding_window, count, partition, partition_all, take_nth, pluck, join, - topk) + diff, topk) from cytoolz.compatibility import range, filter from operator import add, mul @@ -244,14 +244,6 @@ def test_reduceby(): lambda acc, x: acc + x['cost'], projects, 0) == {'CA': 1200000, 'IL': 2100000} - assert reduceby(['state'], - lambda acc, x: acc + x['cost'], - projects, 0) == {('CA',): 1200000, ('IL',): 2100000} - - assert reduceby(['state', 'state'], - lambda acc, x: acc + x['cost'], - projects, 0) == {('CA', 'CA'): 1200000, ('IL', 'IL'): 2100000} - def test_reduce_by_init(): assert reduceby(iseven, add, [1, 2, 3, 4]) == {True: 2 + 4, False: 1 + 3} @@ -419,6 +411,37 @@ def test_outer_join(): assert result == expected +def test_diff(): + assert raises(TypeError, lambda: list(diff())) + assert raises(TypeError, lambda: list(diff([1, 2]))) + assert raises(TypeError, lambda: list(diff([1, 2], 3))) + assert list(diff([1, 2], (1, 2), iter([1, 2]))) == [] + assert list(diff([1, 2, 3], (1, 10, 3), iter([1, 2, 10]))) == [ + (2, 10, 2), (3, 3, 10)] + assert list(diff([1, 2], [10])) == [(1, 10)] + assert list(diff([1, 2], [10], default=None)) == [(1, 10), (2, None)] + # non-variadic usage + assert raises(TypeError, lambda: list(diff([]))) + assert raises(TypeError, lambda: list(diff([[]]))) + assert raises(TypeError, lambda: list(diff([[1, 2]]))) + assert raises(TypeError, lambda: list(diff([[1, 2], 3]))) + assert list(diff([(1, 2), (1, 3)])) == [(2, 3)] + + data1 = [{'cost': 1, 'currency': 'dollar'}, + {'cost': 2, 'currency': 'dollar'}] + + data2 = [{'cost': 100, 'currency': 'yen'}, + {'cost': 300, 'currency': 'yen'}] + + conversions = {'dollar': 1, 'yen': 0.01} + + def indollars(item): + return conversions[item['currency']] * item['cost'] + + list(diff(data1, data2, key=indollars)) == [ + ({'cost': 2, 'currency': 'dollar'}, {'cost': 300, 'currency': 'yen'})] + + def test_topk(): assert topk(2, [4, 1, 5, 2]) == (5, 4) assert topk(2, [4, 1, 5, 2], key=lambda x: -x) == (1, 2) diff --git a/cytoolz/tests/test_none_safe.py b/cytoolz/tests/test_none_safe.py index 6e4d6a4..8570a70 100644 --- a/cytoolz/tests/test_none_safe.py +++ b/cytoolz/tests/test_none_safe.py @@ -282,6 +282,12 @@ def test_itertoolz(): assert raises(TypeError, lambda: topk(3, None)) tested.append('topk') + assert raises(TypeError, lambda: list(diff(None, [1, 2, 3]))) + assert raises(TypeError, lambda: list(diff(None))) + assert raises(TypeError, lambda: list(diff([None]))) + assert raises(TypeError, lambda: list(diff([None, None]))) + tested.append('diff') + s1 = set(tested) s2 = set(cytoolz.itertoolz.__all__) assert s1 == s2, '%s not tested for being None-safe' % ', '.join(s2 - s1) diff --git a/cytoolz/tests/test_recipes.py b/cytoolz/tests/test_recipes.py index 4ec0a9b..13309f4 100644 --- a/cytoolz/tests/test_recipes.py +++ b/cytoolz/tests/test_recipes.py @@ -9,8 +9,6 @@ def test_countby(): assert countby(iseven, [1, 2, 3]) == {True: 1, False: 2} assert countby(len, ['cat', 'dog', 'mouse']) == {3: 2, 5: 1} assert countby(0, ('ab', 'ac', 'bc')) == {'a': 2, 'b': 1} - assert countby([0], ('ab', 'ac', 'bc')) == {('a',): 2, ('b',): 1} - assert countby([0, 0], ('ab', 'ac', 'bc')) == {('a', 'a'): 2, ('b', 'b'): 1} def test_partitionby(): From 35857262e453e684eea002284e4b9e3e23130dde Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 23 May 2015 09:57:14 -0500 Subject: [PATCH 066/146] Add factory keyword functionality to dicttoolz. To perform iteration over any mapping, ``dicttoolz.get_map_iter`` C function was created. It returns a function pointer with a signature that matches PyDict_Next. Hence, iteration is very fast for dict inputs. For dicts, this is significantly faster than Cython's compatible iteration when using "d.iteritems()". There is a small performance penalty when iterating over generic mappings using our method, but I think it is worth it in order to be fast for dicts. The other advantage of using ``get_map_iter`` over ``d.iteritems()`` is we can define our own rules for how we want to iterate over the mapping. cytoolz now matches toolz 0.7.2. woot! --- cytoolz/cpython.pxd | 2 + cytoolz/dicttoolz.pxd | 32 ++- cytoolz/dicttoolz.pyx | 361 ++++++++++++++++++++------------ cytoolz/tests/test_none_safe.py | 12 +- 4 files changed, 258 insertions(+), 149 deletions(-) diff --git a/cytoolz/cpython.pxd b/cytoolz/cpython.pxd index 7dfca1c..d7df24b 100644 --- a/cytoolz/cpython.pxd +++ b/cytoolz/cpython.pxd @@ -8,3 +8,5 @@ cdef extern from "Python.h": PyObject* PtrIter_Next "PyIter_Next"(object o) PyObject* PtrObject_Call "PyObject_Call"(object callable_object, object args, object kw) PyObject* PtrObject_GetItem "PyObject_GetItem"(object o, object key) + int PyDict_Next_Compat "PyDict_Next"(object p, Py_ssize_t *ppos, PyObject* *pkey, PyObject* *pvalue) except -1 + diff --git a/cytoolz/dicttoolz.pxd b/cytoolz/dicttoolz.pxd index c01b4bf..c40c19f 100644 --- a/cytoolz/dicttoolz.pxd +++ b/cytoolz/dicttoolz.pxd @@ -1,34 +1,44 @@ -cdef dict c_merge(object dicts) +from cpython.ref cimport PyObject +# utility functions to perform iteration over dicts or generic mapping +ctypedef int (*f_map_next)(object p, Py_ssize_t *ppos, PyObject* *pkey, PyObject* *pval) except -1 -cdef dict c_merge_with(object func, object dicts) +cdef f_map_next get_map_iter(object d, PyObject* *ptr) except NULL +cdef int PyMapping_Next(object p, Py_ssize_t *ppos, PyObject* *pkey, PyObject* *pval) except -1 -cpdef dict valmap(object func, dict d, object factory=*) +cdef object c_merge(object dicts, object factory=*) -cpdef dict keymap(object func, dict d, object factory=*) +cdef object c_merge_with(object func, object dicts, object factory=*) -cpdef dict itemmap(object func, dict d, object factory=*) +cpdef object valmap(object func, object d, object factory=*) -cpdef dict valfilter(object predicate, dict d, object factory=*) +cpdef object keymap(object func, object d, object factory=*) -cpdef dict keyfilter(object predicate, dict d, object factory=*) +cpdef object itemmap(object func, object d, object factory=*) -cpdef dict itemfilter(object predicate, dict d, object factory=*) +cpdef object valfilter(object predicate, object d, object factory=*) -cpdef dict assoc(dict d, object key, object value, object factory=*) +cpdef object keyfilter(object predicate, object d, object factory=*) -cpdef dict dissoc(dict d, object key) +cpdef object itemfilter(object predicate, object d, object factory=*) -cpdef dict update_in(dict d, object keys, object func, object default=*, object factory=*) + +cpdef object assoc(object d, object key, object value, object factory=*) + + +cpdef object dissoc(object d, object key) + + +cpdef object update_in(object d, object keys, object func, object default=*, object factory=*) cpdef object get_in(object keys, object coll, object default=*, object no_default=*) diff --git a/cytoolz/dicttoolz.pyx b/cytoolz/dicttoolz.pyx index 5b0303e..a10efff 100644 --- a/cytoolz/dicttoolz.pyx +++ b/cytoolz/dicttoolz.pyx @@ -1,24 +1,87 @@ #cython: embedsignature=True -from cpython.dict cimport (PyDict_Check, PyDict_GetItem, PyDict_Merge, - PyDict_New, PyDict_Next, PyDict_SetItem, - PyDict_Update, PyDict_DelItem) +from cpython.dict cimport (PyDict_Check, PyDict_CheckExact, PyDict_GetItem, + PyDict_Merge, PyDict_New, PyDict_Next, + PyDict_SetItem, PyDict_Update, PyDict_DelItem) from cpython.exc cimport PyErr_Clear, PyErr_GivenExceptionMatches, PyErr_Occurred from cpython.list cimport PyList_Append, PyList_New -from cpython.ref cimport PyObject, Py_XDECREF +from cpython.object cimport PyObject_SetItem +from cpython.ref cimport PyObject, Py_DECREF, Py_INCREF, Py_XDECREF +#from cpython.type cimport PyType_Check # Locally defined bindings that differ from `cython.cpython` bindings -from cytoolz.cpython cimport PtrObject_GetItem +from cytoolz.cpython cimport PtrObject_GetItem, PyDict_Next_Compat, PtrIter_Next + +from copy import copy __all__ = ['merge', 'merge_with', 'valmap', 'keymap', 'itemmap', 'valfilter', 'keyfilter', 'itemfilter', 'assoc', 'dissoc', 'get_in', 'update_in'] -cdef dict c_merge(object dicts): - cdef dict rv - rv = PyDict_New() - for d in dicts: - PyDict_Update(rv, d) +cdef int PyMapping_Next(object p, Py_ssize_t *ppos, PyObject* *pkey, PyObject* *pval) except -1: + """Mimic "PyDict_Next" interface, but for any mapping""" + cdef PyObject *obj + obj = PtrIter_Next(p) + if obj is NULL: + return 0 + pkey[0] = (obj)[0] + pval[0] = (obj)[1] + Py_XDECREF(obj) + return 1 + + +cdef f_map_next get_map_iter(object d, PyObject* *ptr) except NULL: + """Return function pointer to perform iteration over object returned in ptr. + + The returned function signature matches "PyDict_Next". If ``d`` is a dict, + then the returned function *is* PyDict_Next, so iteration wil be very fast. + + The object returned through ``ptr`` needs to have its reference count + reduced by one once the caller "owns" the object. + + This function lets us control exactly how iteration should be performed + over a given mapping. The current rules are: + + 1) If ``d`` is exactly a dict, use PyDict_Next + 2) If ``d`` is subtype of dict, use PyMapping_Next. This lets the user + control the order iteration, such as for ordereddict. + 3) If using PyMapping_Next, iterate using ``iteritems`` if possible, + otherwise iterate using ``items``. + + """ + cdef object val + cdef f_map_next rv + if PyDict_CheckExact(d): + val = d + rv = &PyDict_Next_Compat + elif hasattr(d, 'iteritems'): + val = iter(d.iteritems()) + rv = &PyMapping_Next + else: + val = iter(d.items()) + rv = &PyMapping_Next + Py_INCREF(val) + ptr[0] = val + return rv + + +cdef get_factory(name, kwargs): + factory = kwargs.pop('factory', dict) + if kwargs: + raise TypeError("{0}() got an unexpected keyword argument " + "'{1}'".format(name, kwargs.popitem()[0])) + return factory + + +cdef object c_merge(object dicts, object factory=dict): + cdef object rv + rv = factory() + if PyDict_CheckExact(rv): + for d in dicts: + PyDict_Update(rv, d) + else: + for d in dicts: + rv.update(d) return rv @@ -39,27 +102,43 @@ def merge(*dicts, **kwargs): """ if len(dicts) == 1 and not PyDict_Check(dicts[0]): dicts = dicts[0] - return c_merge(dicts) + factory = get_factory('merge', kwargs) + return c_merge(dicts, factory) -cdef dict c_merge_with(object func, object dicts): - cdef dict result, rv, d - cdef list seq - cdef object k, v - cdef PyObject *obj +cdef object c_merge_with(object func, object dicts, object factory=dict): + cdef: + dict result + object rv, d + list seq + f_map_next f + PyObject *obj + PyObject *pkey + PyObject *pval + Py_ssize_t pos + result = PyDict_New() - rv = PyDict_New() + rv = factory() for d in dicts: - for k, v in d.iteritems(): - obj = PyDict_GetItem(result, k) + f = get_map_iter(d, &obj) + d = obj + Py_DECREF(d) + pos = 0 + while f(d, &pos, &pkey, &pval): + obj = PyDict_GetItem(result, pkey) if obj is NULL: seq = PyList_New(0) - PyList_Append(seq, v) - PyDict_SetItem(result, k, seq) + PyList_Append(seq, pval) + PyDict_SetItem(result, pkey, seq) else: - PyList_Append(obj, v) - for k, v in result.iteritems(): - PyDict_SetItem(rv, k, func(v)) + PyList_Append(obj, pval) + + f = get_map_iter(result, &obj) + d = obj + Py_DECREF(d) + pos = 0 + while f(d, &pos, &pkey, &pval): + PyObject_SetItem(rv, pkey, func(pval)) return rv @@ -81,10 +160,11 @@ def merge_with(func, *dicts, **kwargs): """ if len(dicts) == 1 and not PyDict_Check(dicts[0]): dicts = dicts[0] - return c_merge_with(func, dicts) + factory = get_factory('merge_with', kwargs) + return c_merge_with(func, dicts, factory) -cpdef dict valmap(object func, dict d, object factory=dict): +cpdef object valmap(object func, object d, object factory=dict): """ Apply function to values of dictionary @@ -97,22 +177,23 @@ cpdef dict valmap(object func, dict d, object factory=dict): itemmap """ cdef: - dict rv - Py_ssize_t pos - PyObject *k - PyObject *v - - if d is None: - raise TypeError("expected dict, got None") - - rv = PyDict_New() - pos = 0 - while PyDict_Next(d, &pos, &k, &v): - PyDict_SetItem(rv, k, func(v)) + object rv + f_map_next f + PyObject *obj + PyObject *pkey + PyObject *pval + Py_ssize_t pos = 0 + + rv = factory() + f = get_map_iter(d, &obj) + d = obj + Py_DECREF(d) + while f(d, &pos, &pkey, &pval): + rv[pkey] = func(pval) return rv -cpdef dict keymap(object func, dict d, object factory=dict): +cpdef object keymap(object func, object d, object factory=dict): """ Apply function to keys of dictionary @@ -125,22 +206,23 @@ cpdef dict keymap(object func, dict d, object factory=dict): itemmap """ cdef: - dict rv - Py_ssize_t pos - PyObject *k - PyObject *v - - if d is None: - raise TypeError("expected dict, got None") - - rv = PyDict_New() - pos = 0 - while PyDict_Next(d, &pos, &k, &v): - PyDict_SetItem(rv, func(k), v) + object rv + f_map_next f + PyObject *obj + PyObject *pkey + PyObject *pval + Py_ssize_t pos = 0 + + rv = factory() + f = get_map_iter(d, &obj) + d = obj + Py_DECREF(d) + while f(d, &pos, &pkey, &pval): + rv[func(pkey)] = pval return rv -cpdef dict itemmap(object func, dict d, object factory=dict): +cpdef object itemmap(object func, object d, object factory=dict): """ Apply function to items of dictionary @@ -153,24 +235,24 @@ cpdef dict itemmap(object func, dict d, object factory=dict): valmap """ cdef: - dict rv - object newk, newv - Py_ssize_t pos - PyObject *k - PyObject *v - - if d is None: - raise TypeError("expected dict, got None") - - rv = PyDict_New() - pos = 0 - while PyDict_Next(d, &pos, &k, &v): - newk, newv = func((k, v)) - PyDict_SetItem(rv, newk, newv) + object rv, k, v + f_map_next f + PyObject *obj + PyObject *pkey + PyObject *pval + Py_ssize_t pos = 0 + + rv = factory() + f = get_map_iter(d, &obj) + d = obj + Py_DECREF(d) + while f(d, &pos, &pkey, &pval): + k, v = func((pkey, pval)) + rv[k] = v return rv -cpdef dict valfilter(object predicate, dict d, object factory=dict): +cpdef object valfilter(object predicate, object d, object factory=dict): """ Filter items in dictionary by value @@ -185,23 +267,24 @@ cpdef dict valfilter(object predicate, dict d, object factory=dict): valmap """ cdef: - dict rv - Py_ssize_t pos - PyObject *k - PyObject *v - - if d is None: - raise TypeError("expected dict, got None") - - rv = PyDict_New() - pos = 0 - while PyDict_Next(d, &pos, &k, &v): - if predicate(v): - PyDict_SetItem(rv, k, v) + object rv + f_map_next f + PyObject *obj + PyObject *pkey + PyObject *pval + Py_ssize_t pos = 0 + + rv = factory() + f = get_map_iter(d, &obj) + d = obj + Py_DECREF(d) + while f(d, &pos, &pkey, &pval): + if predicate(pval): + rv[pkey] = pval return rv -cpdef dict keyfilter(object predicate, dict d, object factory=dict): +cpdef object keyfilter(object predicate, object d, object factory=dict): """ Filter items in dictionary by key @@ -216,23 +299,24 @@ cpdef dict keyfilter(object predicate, dict d, object factory=dict): keymap """ cdef: - dict rv - Py_ssize_t pos - PyObject *k - PyObject *v - - if d is None: - raise TypeError("expected dict, got None") - - rv = PyDict_New() - pos = 0 - while PyDict_Next(d, &pos, &k, &v): - if predicate(k): - PyDict_SetItem(rv, k, v) + object rv + f_map_next f + PyObject *obj + PyObject *pkey + PyObject *pval + Py_ssize_t pos = 0 + + rv = factory() + f = get_map_iter(d, &obj) + d = obj + Py_DECREF(d) + while f(d, &pos, &pkey, &pval): + if predicate(pkey): + rv[pkey] = pval return rv -cpdef dict itemfilter(object predicate, dict d, object factory=dict): +cpdef object itemfilter(object predicate, object d, object factory=dict): """ Filter items in dictionary by item @@ -250,23 +334,26 @@ cpdef dict itemfilter(object predicate, dict d, object factory=dict): itemmap """ cdef: - dict rv - Py_ssize_t pos - PyObject *k - PyObject *v - - if d is None: - raise TypeError("expected dict, got None") - - rv = PyDict_New() - pos = 0 - while PyDict_Next(d, &pos, &k, &v): - if predicate((k, v)): - PyDict_SetItem(rv, k, v) + object rv, k, v + f_map_next f + PyObject *obj + PyObject *pkey + PyObject *pval + Py_ssize_t pos = 0 + + rv = factory() + f = get_map_iter(d, &obj) + d = obj + Py_DECREF(d) + while f(d, &pos, &pkey, &pval): + k = pkey + v = pval + if predicate((k, v)): + rv[k] = v return rv -cpdef dict assoc(dict d, object key, object value, object factory=dict): +cpdef object assoc(object d, object key, object value, object factory=dict): """ Return a new dict with new key value pair @@ -277,13 +364,17 @@ cpdef dict assoc(dict d, object key, object value, object factory=dict): >>> assoc({'x': 1}, 'y', 3) # doctest: +SKIP {'x': 1, 'y': 3} """ - cdef dict rv - rv = d.copy() - PyDict_SetItem(rv, key, value) + cdef object rv + rv = factory() + if PyDict_CheckExact(rv): + PyDict_Update(rv, d) + else: + rv.update(d) + rv[key] = value return rv -cpdef dict dissoc(dict d, object key): +cpdef object dissoc(object d, object key): """ Return a new dict with the given key removed. @@ -293,13 +384,13 @@ cpdef dict dissoc(dict d, object key): >>> dissoc({'x': 1, 'y': 2}, 'y') {'x': 1} """ - cdef dict rv - rv = d.copy() - PyDict_DelItem(rv, key) + cdef object rv + rv = copy(d) + del rv[key] return rv -cpdef dict update_in(dict d, object keys, object func, object default=None, object factory=dict): +cpdef object update_in(object d, object keys, object func, object default=None, object factory=dict): """ Update value in a (potentially) nested dictionary @@ -335,30 +426,35 @@ cpdef dict update_in(dict d, object keys, object func, object default=None, obje {1: 'foo', 2: {3: {4: 1}}} """ cdef object prevkey, key - cdef dict rv, inner, dtemp - cdef PyObject *obj + cdef object rv, inner, dtemp prevkey, keys = keys[0], keys[1:] - rv = d.copy() + rv = factory() + if PyDict_CheckExact(rv): + PyDict_Update(rv, d) + else: + rv.update(d) inner = rv for key in keys: - obj = PyDict_GetItem(d, prevkey) - if obj is NULL: - d = PyDict_New() - dtemp = d + if prevkey in d: + d = d[prevkey] + dtemp = factory() + if PyDict_CheckExact(dtemp): + PyDict_Update(dtemp, d) + else: + dtemp.update(d) else: - d = obj - dtemp = d.copy() - PyDict_SetItem(inner, prevkey, dtemp) + d = factory() + dtemp = d + inner[prevkey] = dtemp prevkey = key inner = dtemp - obj = PyDict_GetItem(d, prevkey) - if obj is NULL: - key = func(default) + if prevkey in d: + key = func(d[prevkey]) else: - key = func(obj) - PyDict_SetItem(inner, prevkey, key) + key = func(default) + inner[prevkey] = key return rv @@ -410,3 +506,4 @@ cpdef object get_in(object keys, object coll, object default=None, object no_def Py_XDECREF(obj) coll = obj return coll + diff --git a/cytoolz/tests/test_none_safe.py b/cytoolz/tests/test_none_safe.py index 8570a70..f7ad5ec 100644 --- a/cytoolz/tests/test_none_safe.py +++ b/cytoolz/tests/test_none_safe.py @@ -56,13 +56,13 @@ def test_dicttoolz(): tested.append('get_in') assert raises(TypeError, lambda: keyfilter(None, {1: 2})) - assert raises(TypeError, lambda: keyfilter(identity, None)) + assert raises((AttributeError, TypeError), lambda: keyfilter(identity, None)) tested.append('keyfilter') # XXX assert (raises(TypeError, lambda: keymap(None, {1: 2})) or keymap(None, {1: 2}) == {(1,): 2}) - assert raises(TypeError, lambda: keymap(identity, None)) + assert raises((AttributeError, TypeError), lambda: keymap(identity, None)) tested.append('keymap') assert raises(TypeError, lambda: merge(None)) @@ -82,22 +82,22 @@ def test_dicttoolz(): tested.append('update_in') assert raises(TypeError, lambda: valfilter(None, {1: 2})) - assert raises(TypeError, lambda: valfilter(identity, None)) + assert raises((AttributeError, TypeError), lambda: valfilter(identity, None)) tested.append('valfilter') # XXX assert (raises(TypeError, lambda: valmap(None, {1: 2})) or valmap(None, {1: 2}) == {1: (2,)}) - assert raises(TypeError, lambda: valmap(identity, None)) + assert raises((AttributeError, TypeError), lambda: valmap(identity, None)) tested.append('valmap') assert (raises(TypeError, lambda: itemmap(None, {1: 2})) or itemmap(None, {1: 2}) == {1: (2,)}) - assert raises(TypeError, lambda: itemmap(identity, None)) + assert raises((AttributeError, TypeError), lambda: itemmap(identity, None)) tested.append('itemmap') assert raises(TypeError, lambda: itemfilter(None, {1: 2})) - assert raises(TypeError, lambda: itemfilter(identity, None)) + assert raises((AttributeError, TypeError), lambda: itemfilter(identity, None)) tested.append('itemfilter') s1 = set(tested) From 8196cd97606abf5def3ac8af9328cb696779da10 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 23 May 2015 10:43:03 -0500 Subject: [PATCH 067/146] Bump version to 0.7.3 in conda recipe --- conda.recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 5c73925..b535090 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: cytoolz - version: "0.7.2" + version: "0.7.3" build: number: {{environ.get('BINSTAR_BUILD', 1)}} From c9f6b0616cd773b0c841c630119359fe87106f3a Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 23 May 2015 11:19:46 -0500 Subject: [PATCH 068/146] Bump to next dev version. Just released 0.7.3. --- cytoolz/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cytoolz/_version.py b/cytoolz/_version.py index a86b93a..5816fa9 100644 --- a/cytoolz/_version.py +++ b/cytoolz/_version.py @@ -1,2 +1,2 @@ -__version__ = '0.7.3' +__version__ = '0.7.4dev' __toolz_version__ = '0.7.2' From 92b6d30323a818ed280640095940b127a6736479 Mon Sep 17 00:00:00 2001 From: llllllllll Date: Fri, 31 Jul 2015 20:15:21 -0400 Subject: [PATCH 069/146] ENH: accumulate start --- cytoolz/itertoolz.pxd | 1 + cytoolz/itertoolz.pyx | 16 ++++++++-------- cytoolz/tests/test_itertoolz.py | 7 +++++++ cytoolz/utils.pyx | 16 +++++++++++++++- 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index 8b98424..67edd85 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -7,6 +7,7 @@ cdef class accumulate: cdef object binop cdef object iter_seq cdef object result + cdef object start cpdef dict groupby(object key, object seq) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index cbb8c3d..4497873 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -17,6 +17,7 @@ from heapq import heapify, heappop, heapreplace from itertools import chain, islice from operator import itemgetter from cytoolz.compatibility import map, zip, zip_longest +from cytoolz.utils import no_default __all__ = ['remove', 'accumulate', 'groupby', 'merge_sorted', 'interleave', @@ -81,21 +82,23 @@ cdef class accumulate: See Also: itertools.accumulate : In standard itertools for Python 3.2+ """ - def __cinit__(self, object binop, object seq): + def __cinit__(self, object binop, object seq, object start=no_default): self.binop = binop self.iter_seq = iter(seq) self.result = self # sentinel + self.start = start def __iter__(self): return self def __next__(self): - cdef object val - val = next(self.iter_seq) if self.result is self: - self.result = val + if self.start is not no_default: + self.result = self.start + else: + self.result = next(self.iter_seq) else: - self.result = self.binop(self.result, val) + self.result = self.binop(self.result, next(self.iter_seq)) return self.result @@ -571,9 +574,6 @@ cpdef object nth(Py_ssize_t n, object seq): return next(seq) -no_default = '__no__default__' - - cpdef object last(object seq): """ The last element in a sequence diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index fcdd53b..cf1895c 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -266,6 +266,13 @@ def test_iterate(): def test_accumulate(): assert list(accumulate(add, [1, 2, 3, 4, 5])) == [1, 3, 6, 10, 15] assert list(accumulate(mul, [1, 2, 3, 4, 5])) == [1, 2, 6, 24, 120] + assert list(accumulate(add, [1, 2, 3, 4, 5], -1)) == [-1, 0, 2, 5, 9, 14] + + def binop(a, b): + raise AssertionError('binop should not be called') + + start = object() + assert list(accumulate(binop, [], start)) == [start] def test_accumulate_works_on_consumable_iterables(): diff --git a/cytoolz/utils.pyx b/cytoolz/utils.pyx index 0d0ba31..6297cdb 100644 --- a/cytoolz/utils.pyx +++ b/cytoolz/utils.pyx @@ -16,7 +16,21 @@ def raises(err, lamda): return True -no_default = '__no__default__' +@object.__new__ +class no_default(object): + def __new__(self): + raise TypeError("cannot create 'no_default' instances") + + def __str__(self): + return '' + __repr__ = __str__ + + +try: + # Attempt to get the no_default sentinel object from toolz + from toolz.utils import no_default +except ImportError: + pass def include_dirs(): From b5e3a394ac3d4517e97c05b08e17fdd40371a338 Mon Sep 17 00:00:00 2001 From: llllllllll Date: Fri, 31 Jul 2015 21:08:50 -0400 Subject: [PATCH 070/146] ENH: adds curried operator --- cytoolz/{curried.py => curried/__init__.py} | 105 ++++----- .../exceptions.pyx} | 4 + cytoolz/curried/operator.py | 207 ++++++++++++++++++ cytoolz/tests/test_curried_toolzlike.py | 4 +- etc/generate_curried.py | 80 +++++++ setup.py | 2 +- 6 files changed, 334 insertions(+), 68 deletions(-) rename cytoolz/{curried.py => curried/__init__.py} (57%) rename cytoolz/{curried_exceptions.pyx => curried/exceptions.pyx} (96%) create mode 100644 cytoolz/curried/operator.py create mode 100644 etc/generate_curried.py diff --git a/cytoolz/curried.py b/cytoolz/curried/__init__.py similarity index 57% rename from cytoolz/curried.py rename to cytoolz/curried/__init__.py index 722b05b..ae60c5c 100644 --- a/cytoolz/curried.py +++ b/cytoolz/curried/__init__.py @@ -1,83 +1,58 @@ """ -Alternate namespece for cytoolz such that all functions are curried +THIS FILE IS AUTOMATICALLY GENERATED -Currying provides implicit partial evaluation of all functions - -Example: - - Get usually requires two arguments, an index and a collection - >>> from cytoolz.curried import get - >>> get(0, ('a', 'b')) - 'a' - - When we use it in higher order functions we often want to pass a partially - evaluated form - >>> data = [(1, 2), (11, 22), (111, 222)] - >>> list(map(lambda seq: get(0, seq), data)) - [1, 11, 111] - - The curried version allows simple expression of partial evaluation - >>> list(map(get(0), data)) - [1, 11, 111] - -See Also: - cytoolz.functoolz.curry +Please make all edits in etc/generate_curried.py and re-run +cytoolz/tests/test_curried_toolzlike.py """ import cytoolz from cytoolz import * -from cytoolz.curried_exceptions import * +from .operator import * -# Here is the recipe used to create the list below -# (and "cytoolz/tests/test_curried_toolzlike.py" verifies the list is correct): -# -# import toolz -# import toolz.curried -# -# for item in sorted(key for key, val in toolz.curried.__dict__.items() -# if isinstance(val, toolz.curry)): -# print '%s = cytoolz.curry(%s)' % (item, item) - -accumulate = cytoolz.curry(accumulate) -assoc = cytoolz.curry(assoc) -cons = cytoolz.curry(cons) -countby = cytoolz.curry(countby) -dissoc = cytoolz.curry(dissoc) -do = cytoolz.curry(do) -drop = cytoolz.curry(drop) -filter = cytoolz.curry(filter) get = cytoolz.curry(get) +tail = cytoolz.curry(tail) +pluck = cytoolz.curry(pluck) +update_in = cytoolz.curry(update_in) get_in = cytoolz.curry(get_in) -groupby = cytoolz.curry(groupby) -interleave = cytoolz.curry(interleave) -interpose = cytoolz.curry(interpose) -itemfilter = cytoolz.curry(itemfilter) -itemmap = cytoolz.curry(itemmap) iterate = cytoolz.curry(iterate) -join = cytoolz.curry(join) -keyfilter = cytoolz.curry(keyfilter) -keymap = cytoolz.curry(keymap) -map = cytoolz.curry(map) -mapcat = cytoolz.curry(mapcat) -memoize = cytoolz.curry(memoize) -merge = cytoolz.curry(merge) +partitionby = cytoolz.curry(partitionby) +unique = cytoolz.curry(unique) +valfilter = cytoolz.curry(valfilter) merge_with = cytoolz.curry(merge_with) +take = cytoolz.curry(take) nth = cytoolz.curry(nth) +take_nth = cytoolz.curry(take_nth) +interpose = cytoolz.curry(interpose) +accumulate = cytoolz.curry(accumulate) +map = cytoolz.curry(map) partition = cytoolz.curry(partition) -partition_all = cytoolz.curry(partition_all) -partitionby = cytoolz.curry(partitionby) -pluck = cytoolz.curry(pluck) -reduce = cytoolz.curry(reduce) +valmap = cytoolz.curry(valmap) +memoize = cytoolz.curry(memoize) +mapcat = cytoolz.curry(mapcat) reduceby = cytoolz.curry(reduceby) +countby = cytoolz.curry(countby) +sorted = cytoolz.curry(sorted) +drop = cytoolz.curry(drop) remove = cytoolz.curry(remove) +merge = cytoolz.curry(merge) +itemfilter = cytoolz.curry(itemfilter) sliding_window = cytoolz.curry(sliding_window) -sorted = cytoolz.curry(sorted) -tail = cytoolz.curry(tail) -take = cytoolz.curry(take) -take_nth = cytoolz.curry(take_nth) +reduce = cytoolz.curry(reduce) +keyfilter = cytoolz.curry(keyfilter) +do = cytoolz.curry(do) topk = cytoolz.curry(topk) -unique = cytoolz.curry(unique) -update_in = cytoolz.curry(update_in) -valfilter = cytoolz.curry(valfilter) -valmap = cytoolz.curry(valmap) +partition_all = cytoolz.curry(partition_all) +assoc = cytoolz.curry(assoc) +join = cytoolz.curry(join) +filter = cytoolz.curry(filter) +itemmap = cytoolz.curry(itemmap) +interleave = cytoolz.curry(interleave) +cons = cytoolz.curry(cons) +groupby = cytoolz.curry(groupby) +keymap = cytoolz.curry(keymap) +dissoc = cytoolz.curry(dissoc) + +from .exceptions import * +del cytoolz + diff --git a/cytoolz/curried_exceptions.pyx b/cytoolz/curried/exceptions.pyx similarity index 96% rename from cytoolz/curried_exceptions.pyx rename to cytoolz/curried/exceptions.pyx index 3a9e98f..75ffe66 100644 --- a/cytoolz/curried_exceptions.pyx +++ b/cytoolz/curried/exceptions.pyx @@ -1,10 +1,13 @@ #cython: embedsignature=True +from cytoolz import curry + from cpython.dict cimport PyDict_Check from cytoolz.dicttoolz cimport c_merge, c_merge_with __all__ = ['merge', 'merge_with'] +@curry def merge(*dicts, **kwargs): if len(dicts) == 0: raise TypeError() @@ -13,6 +16,7 @@ def merge(*dicts, **kwargs): return c_merge(dicts) +@curry def merge_with(func, *dicts, **kwargs): """ Merge dictionaries and apply function to combined values diff --git a/cytoolz/curried/operator.py b/cytoolz/curried/operator.py new file mode 100644 index 0000000..6a21997 --- /dev/null +++ b/cytoolz/curried/operator.py @@ -0,0 +1,207 @@ +""" +THIS FILE IS AUTOMATICALLY GENERATED + +Please make all edits in etc/generate_curried.py and re-run +cytoolz/tests/test_curried_toolzlike.py +""" + +from operator import ( + __eq__, + pos, + floordiv, + __itruediv__, + add, + countOf, + imul, + concat, + __ilshift__, + __getitem__, + abs, + not_, + __rshift__, + indexOf, + __irshift__, + __builtins__, + delitem, + setitem, + eq, + __pow__, + __ge__, + __lt__, + iadd, + __index__, + gt, + iand, + length_hint, + and_, + irshift, + xor, + getitem, + truediv, + imod, + __lshift__, + ge, + __invert__, + __ne__, + index, + __and__, + __mul__, + ior, + __delitem__, + __package__, + __inv__, + methodcaller, + mul, + __pos__, + rshift, + __iadd__, + iconcat, + inv, + itruediv, + __imod__, + ilshift, + contains, + pow, + sub, + ipow, + __floordiv__, + __ior__, + __cached__, + __ipow__, + __xor__, + __truediv__, + _abs, + __name__, + ifloordiv, + attrgetter, + __contains__, + __file__, + __abs__, + __doc__, + __loader__, + __not__, + __iconcat__, + __ixor__, + __setitem__, + lshift, + mod, + le, + invert, + __neg__, + isub, + is_not, + __ifloordiv__, + or_, + __concat__, + __spec__, + __le__, + lt, + __add__, + __iand__, + __sub__, + __gt__, + neg, + __mod__, + ne, + __imul__, + itemgetter, + __or__, + ixor, + __isub__, + is_, + truth, +) + +import cytoolz + +__eq__ = cytoolz.curry(__eq__) +floordiv = cytoolz.curry(floordiv) +__itruediv__ = cytoolz.curry(__itruediv__) +add = cytoolz.curry(add) +countOf = cytoolz.curry(countOf) +imul = cytoolz.curry(imul) +concat = cytoolz.curry(concat) +__ilshift__ = cytoolz.curry(__ilshift__) +__getitem__ = cytoolz.curry(__getitem__) +__rshift__ = cytoolz.curry(__rshift__) +indexOf = cytoolz.curry(indexOf) +__irshift__ = cytoolz.curry(__irshift__) +delitem = cytoolz.curry(delitem) +setitem = cytoolz.curry(setitem) +eq = cytoolz.curry(eq) +__pow__ = cytoolz.curry(__pow__) +__ge__ = cytoolz.curry(__ge__) +__lt__ = cytoolz.curry(__lt__) +iadd = cytoolz.curry(iadd) +__index__ = cytoolz.curry(__index__) +gt = cytoolz.curry(gt) +iand = cytoolz.curry(iand) +length_hint = cytoolz.curry(length_hint) +and_ = cytoolz.curry(and_) +irshift = cytoolz.curry(irshift) +xor = cytoolz.curry(xor) +getitem = cytoolz.curry(getitem) +truediv = cytoolz.curry(truediv) +imod = cytoolz.curry(imod) +__lshift__ = cytoolz.curry(__lshift__) +ge = cytoolz.curry(ge) +__invert__ = cytoolz.curry(__invert__) +__ne__ = cytoolz.curry(__ne__) +__and__ = cytoolz.curry(__and__) +__mul__ = cytoolz.curry(__mul__) +ior = cytoolz.curry(ior) +__delitem__ = cytoolz.curry(__delitem__) +__inv__ = cytoolz.curry(__inv__) +methodcaller = cytoolz.curry(methodcaller) +mul = cytoolz.curry(mul) +__pos__ = cytoolz.curry(__pos__) +rshift = cytoolz.curry(rshift) +__iadd__ = cytoolz.curry(__iadd__) +iconcat = cytoolz.curry(iconcat) +itruediv = cytoolz.curry(itruediv) +__imod__ = cytoolz.curry(__imod__) +ilshift = cytoolz.curry(ilshift) +contains = cytoolz.curry(contains) +pow = cytoolz.curry(pow) +sub = cytoolz.curry(sub) +ipow = cytoolz.curry(ipow) +__floordiv__ = cytoolz.curry(__floordiv__) +__ior__ = cytoolz.curry(__ior__) +__ipow__ = cytoolz.curry(__ipow__) +__xor__ = cytoolz.curry(__xor__) +__truediv__ = cytoolz.curry(__truediv__) +_abs = cytoolz.curry(_abs) +ifloordiv = cytoolz.curry(ifloordiv) +attrgetter = cytoolz.curry(attrgetter) +__contains__ = cytoolz.curry(__contains__) +__abs__ = cytoolz.curry(__abs__) +__not__ = cytoolz.curry(__not__) +__iconcat__ = cytoolz.curry(__iconcat__) +__ixor__ = cytoolz.curry(__ixor__) +__setitem__ = cytoolz.curry(__setitem__) +lshift = cytoolz.curry(lshift) +mod = cytoolz.curry(mod) +le = cytoolz.curry(le) +__neg__ = cytoolz.curry(__neg__) +isub = cytoolz.curry(isub) +is_not = cytoolz.curry(is_not) +__ifloordiv__ = cytoolz.curry(__ifloordiv__) +or_ = cytoolz.curry(or_) +__concat__ = cytoolz.curry(__concat__) +__le__ = cytoolz.curry(__le__) +lt = cytoolz.curry(lt) +__add__ = cytoolz.curry(__add__) +__iand__ = cytoolz.curry(__iand__) +__sub__ = cytoolz.curry(__sub__) +__gt__ = cytoolz.curry(__gt__) +__mod__ = cytoolz.curry(__mod__) +ne = cytoolz.curry(ne) +__imul__ = cytoolz.curry(__imul__) +itemgetter = cytoolz.curry(itemgetter) +__or__ = cytoolz.curry(__or__) +ixor = cytoolz.curry(ixor) +__isub__ = cytoolz.curry(__isub__) +is_ = cytoolz.curry(is_) + +del cytoolz + diff --git a/cytoolz/tests/test_curried_toolzlike.py b/cytoolz/tests/test_curried_toolzlike.py index d5a4fa5..7fdcdb8 100644 --- a/cytoolz/tests/test_curried_toolzlike.py +++ b/cytoolz/tests/test_curried_toolzlike.py @@ -18,7 +18,7 @@ def test_toolzcurry_is_class(): def test_cytoolz_like_toolz(): import toolz import toolz.curried - for key, val in toolz.curried.__dict__.items(): + for key, val in vars(toolz.curried).items(): if isinstance(val, toolz.curry): if val.func is toolz.curry: # XXX: Python 3.4 work-around! continue @@ -32,7 +32,7 @@ def test_cytoolz_like_toolz(): def test_toolz_like_cytoolz(): import toolz import toolz.curried - for key, val in cytoolz.curried.__dict__.items(): + for key, val in vars(cytoolz.curried).items(): if isinstance(val, cytoolz.curry): assert hasattr(toolz.curried, key), ( 'cytoolz.curried.%s should not exist' % key) diff --git a/etc/generate_curried.py b/etc/generate_curried.py new file mode 100644 index 0000000..0b3c4ce --- /dev/null +++ b/etc/generate_curried.py @@ -0,0 +1,80 @@ +import toolz +import toolz.curried +from toolz.curried import operator as cop + + +init_template = '''\ +""" +THIS FILE IS AUTOMATICALLY GENERATED + +Please make all edits in etc/generate_curried.py and re-run +cytoolz/tests/test_curried_toolzlike.py +""" + +import cytoolz +from cytoolz import * +from .operator import * + + +{curried} + +from .exceptions import * +del cytoolz +''' + + +def gen_init(): + tc = tuple( + (k, v) for k, v in vars(toolz.curried).items() + if isinstance(v, toolz.curry) and 'operator' not in v.func.__module__ + ) + return init_template.format( + curried='\n'.join( + '{0} = cytoolz.curry({0})'.format(k) for k, v in tc + ), + ) + + +op_template = '''\ +""" +THIS FILE IS AUTOMATICALLY GENERATED + +Please make all edits in etc/generate_curried.py and re-run +cytoolz/tests/test_curried_toolzlike.py +""" + +from operator import ( + {imports} +) + +import cytoolz + +{curried} + +del cytoolz +''' + + +def gen_op(): + return op_template.format( + imports='\n '.join(k + ',' for k in vars(cop)), + curried='\n'.join( + '{0} = cytoolz.curry({0})'.format(k) + for k, v in vars(cop).items() if isinstance(v, toolz.curry) + ) + ) + + +def main(argv): + if len(argv) == 1 or argv[1].lower() == 'init': + fn = gen_init + elif argv[1].lower() in ('op', 'operator'): + fn = gen_op + else: + raise ValueError('%s is not a valid argument' % argv[1]) + print(fn()) + + +if __name__ == '__main__': + from sys import argv + main(argv) diff --git a/setup.py b/setup.py index 9195f23..3663b7f 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ ext_modules = [] for modname in ['dicttoolz', 'functoolz', 'itertoolz', - 'curried_exceptions', 'recipes', 'utils']: + 'curried/exceptions', 'recipes', 'utils']: ext_modules.append(Extension('cytoolz.' + modname, ['cytoolz/' + modname + suffix])) From 657a167bc2d93bcb452714178a91c08bfd52943f Mon Sep 17 00:00:00 2001 From: llllllllll Date: Sat, 1 Aug 2015 14:32:25 -0400 Subject: [PATCH 071/146] MAINT: PY2 compat --- cytoolz/curried/__init__.py | 65 ++++--- cytoolz/curried/operator.py | 349 ++++++++++++++++++++---------------- cytoolz/functoolz.pyx | 5 +- etc/generate_curried.py | 10 +- 4 files changed, 234 insertions(+), 195 deletions(-) diff --git a/cytoolz/curried/__init__.py b/cytoolz/curried/__init__.py index ae60c5c..03c6581 100644 --- a/cytoolz/curried/__init__.py +++ b/cytoolz/curried/__init__.py @@ -4,54 +4,53 @@ Please make all edits in etc/generate_curried.py and re-run cytoolz/tests/test_curried_toolzlike.py """ - import cytoolz from cytoolz import * -from .operator import * +from . import operator -get = cytoolz.curry(get) -tail = cytoolz.curry(tail) -pluck = cytoolz.curry(pluck) -update_in = cytoolz.curry(update_in) -get_in = cytoolz.curry(get_in) +reduceby = cytoolz.curry(reduceby) +itemmap = cytoolz.curry(itemmap) iterate = cytoolz.curry(iterate) -partitionby = cytoolz.curry(partitionby) -unique = cytoolz.curry(unique) -valfilter = cytoolz.curry(valfilter) +update_in = cytoolz.curry(update_in) +dissoc = cytoolz.curry(dissoc) +tail = cytoolz.curry(tail) +do = cytoolz.curry(do) merge_with = cytoolz.curry(merge_with) -take = cytoolz.curry(take) -nth = cytoolz.curry(nth) -take_nth = cytoolz.curry(take_nth) -interpose = cytoolz.curry(interpose) +get = cytoolz.curry(get) +itemfilter = cytoolz.curry(itemfilter) accumulate = cytoolz.curry(accumulate) -map = cytoolz.curry(map) -partition = cytoolz.curry(partition) -valmap = cytoolz.curry(valmap) -memoize = cytoolz.curry(memoize) -mapcat = cytoolz.curry(mapcat) -reduceby = cytoolz.curry(reduceby) -countby = cytoolz.curry(countby) sorted = cytoolz.curry(sorted) +unique = cytoolz.curry(unique) +join = cytoolz.curry(join) drop = cytoolz.curry(drop) remove = cytoolz.curry(remove) -merge = cytoolz.curry(merge) -itemfilter = cytoolz.curry(itemfilter) -sliding_window = cytoolz.curry(sliding_window) +interleave = cytoolz.curry(interleave) +assoc = cytoolz.curry(assoc) +cons = cytoolz.curry(cons) +map = cytoolz.curry(map) reduce = cytoolz.curry(reduce) +mapcat = cytoolz.curry(mapcat) +countby = cytoolz.curry(countby) +take_nth = cytoolz.curry(take_nth) +merge = cytoolz.curry(merge) keyfilter = cytoolz.curry(keyfilter) -do = cytoolz.curry(do) +take = cytoolz.curry(take) +memoize = cytoolz.curry(memoize) +partitionby = cytoolz.curry(partitionby) +get_in = cytoolz.curry(get_in) +valmap = cytoolz.curry(valmap) +interpose = cytoolz.curry(interpose) +pluck = cytoolz.curry(pluck) +nth = cytoolz.curry(nth) topk = cytoolz.curry(topk) -partition_all = cytoolz.curry(partition_all) -assoc = cytoolz.curry(assoc) -join = cytoolz.curry(join) +sliding_window = cytoolz.curry(sliding_window) +partition = cytoolz.curry(partition) filter = cytoolz.curry(filter) -itemmap = cytoolz.curry(itemmap) -interleave = cytoolz.curry(interleave) -cons = cytoolz.curry(cons) -groupby = cytoolz.curry(groupby) keymap = cytoolz.curry(keymap) -dissoc = cytoolz.curry(dissoc) +partition_all = cytoolz.curry(partition_all) +valfilter = cytoolz.curry(valfilter) +groupby = cytoolz.curry(groupby) from .exceptions import * del cytoolz diff --git a/cytoolz/curried/operator.py b/cytoolz/curried/operator.py index 6a21997..c65b61b 100644 --- a/cytoolz/curried/operator.py +++ b/cytoolz/curried/operator.py @@ -4,204 +4,237 @@ Please make all edits in etc/generate_curried.py and re-run cytoolz/tests/test_curried_toolzlike.py """ +from __future__ import absolute_import from operator import ( - __eq__, - pos, - floordiv, - __itruediv__, - add, + iand, countOf, - imul, - concat, - __ilshift__, - __getitem__, - abs, - not_, - __rshift__, - indexOf, - __irshift__, - __builtins__, - delitem, - setitem, - eq, - __pow__, - __ge__, - __lt__, - iadd, - __index__, gt, - iand, - length_hint, - and_, - irshift, - xor, - getitem, - truediv, - imod, __lshift__, + __concat__, + pow, + imul, + __iand__, ge, - __invert__, - __ne__, - index, - __and__, - __mul__, - ior, - __delitem__, - __package__, + __irshift__, + setslice, + lshift, + ipow, + __lt__, + sequenceIncludes, __inv__, - methodcaller, - mul, + __rshift__, + le, + __getslice__, __pos__, - rshift, - __iadd__, - iconcat, - inv, - itruediv, - __imod__, - ilshift, + invert, contains, - pow, - sub, - ipow, - __floordiv__, - __ior__, - __cached__, - __ipow__, - __xor__, - __truediv__, - _abs, - __name__, - ifloordiv, - attrgetter, - __contains__, - __file__, + lt, __abs__, - __doc__, - __loader__, - __not__, + isSequenceType, + add, + delslice, + _compare_digest, + concat, + irshift, __iconcat__, __ixor__, - __setitem__, - lshift, - mod, - le, - invert, - __neg__, + __doc__, + __itruediv__, + rshift, + __getitem__, isub, - is_not, - __ifloordiv__, - or_, - __concat__, - __spec__, + __file__, + pos, + __irepeat__, + __pow__, + iconcat, + __gt__, + isCallable, + __eq__, + mod, + __delitem__, + __mod__, + and_, + __iadd__, + isMappingType, + setitem, __le__, - lt, - __add__, - __iand__, + truth, + __floordiv__, __sub__, - __gt__, + div, + __ge__, + ifloordiv, + __ipow__, + ixor, + __isub__, + inv, + not_, + __and__, + __truediv__, + repeat, + __repeat__, + __imod__, + eq, + iadd, + index, + __delslice__, + xor, + sub, + __contains__, neg, - __mod__, + getslice, ne, + idiv, + __package__, + indexOf, + attrgetter, + abs, + __invert__, + methodcaller, + truediv, + mul, + __neg__, + __ne__, + irepeat, + is_, + getitem, + __ifloordiv__, + __idiv__, + isNumberType, + ior, + __setitem__, + __or__, + __add__, + __name__, + ilshift, + __ilshift__, + or_, __imul__, + is_not, + __not__, + __setslice__, + imod, + itruediv, + __xor__, + __ior__, + __div__, + floordiv, + __mul__, + __index__, itemgetter, - __or__, - ixor, - __isub__, - is_, - truth, + delitem, ) import cytoolz -__eq__ = cytoolz.curry(__eq__) -floordiv = cytoolz.curry(floordiv) -__itruediv__ = cytoolz.curry(__itruediv__) -add = cytoolz.curry(add) +iand = cytoolz.curry(iand) countOf = cytoolz.curry(countOf) -imul = cytoolz.curry(imul) -concat = cytoolz.curry(concat) -__ilshift__ = cytoolz.curry(__ilshift__) -__getitem__ = cytoolz.curry(__getitem__) -__rshift__ = cytoolz.curry(__rshift__) -indexOf = cytoolz.curry(indexOf) -__irshift__ = cytoolz.curry(__irshift__) -delitem = cytoolz.curry(delitem) -setitem = cytoolz.curry(setitem) -eq = cytoolz.curry(eq) -__pow__ = cytoolz.curry(__pow__) -__ge__ = cytoolz.curry(__ge__) -__lt__ = cytoolz.curry(__lt__) -iadd = cytoolz.curry(iadd) -__index__ = cytoolz.curry(__index__) gt = cytoolz.curry(gt) -iand = cytoolz.curry(iand) -length_hint = cytoolz.curry(length_hint) -and_ = cytoolz.curry(and_) -irshift = cytoolz.curry(irshift) -xor = cytoolz.curry(xor) -getitem = cytoolz.curry(getitem) -truediv = cytoolz.curry(truediv) -imod = cytoolz.curry(imod) __lshift__ = cytoolz.curry(__lshift__) +__concat__ = cytoolz.curry(__concat__) +pow = cytoolz.curry(pow) +imul = cytoolz.curry(imul) +__iand__ = cytoolz.curry(__iand__) ge = cytoolz.curry(ge) -__invert__ = cytoolz.curry(__invert__) -__ne__ = cytoolz.curry(__ne__) -__and__ = cytoolz.curry(__and__) -__mul__ = cytoolz.curry(__mul__) -ior = cytoolz.curry(ior) -__delitem__ = cytoolz.curry(__delitem__) +__irshift__ = cytoolz.curry(__irshift__) +setslice = cytoolz.curry(setslice) +lshift = cytoolz.curry(lshift) +ipow = cytoolz.curry(ipow) +__lt__ = cytoolz.curry(__lt__) +sequenceIncludes = cytoolz.curry(sequenceIncludes) __inv__ = cytoolz.curry(__inv__) -methodcaller = cytoolz.curry(methodcaller) -mul = cytoolz.curry(mul) +__rshift__ = cytoolz.curry(__rshift__) +le = cytoolz.curry(le) +__getslice__ = cytoolz.curry(__getslice__) __pos__ = cytoolz.curry(__pos__) +contains = cytoolz.curry(contains) +lt = cytoolz.curry(lt) +__abs__ = cytoolz.curry(__abs__) +isSequenceType = cytoolz.curry(isSequenceType) +add = cytoolz.curry(add) +delslice = cytoolz.curry(delslice) +_compare_digest = cytoolz.curry(_compare_digest) +concat = cytoolz.curry(concat) +irshift = cytoolz.curry(irshift) +__iconcat__ = cytoolz.curry(__iconcat__) +__ixor__ = cytoolz.curry(__ixor__) +__itruediv__ = cytoolz.curry(__itruediv__) rshift = cytoolz.curry(rshift) -__iadd__ = cytoolz.curry(__iadd__) +__getitem__ = cytoolz.curry(__getitem__) +isub = cytoolz.curry(isub) +__irepeat__ = cytoolz.curry(__irepeat__) +__pow__ = cytoolz.curry(__pow__) iconcat = cytoolz.curry(iconcat) -itruediv = cytoolz.curry(itruediv) -__imod__ = cytoolz.curry(__imod__) -ilshift = cytoolz.curry(ilshift) -contains = cytoolz.curry(contains) -pow = cytoolz.curry(pow) -sub = cytoolz.curry(sub) -ipow = cytoolz.curry(ipow) +__gt__ = cytoolz.curry(__gt__) +isCallable = cytoolz.curry(isCallable) +__eq__ = cytoolz.curry(__eq__) +mod = cytoolz.curry(mod) +__delitem__ = cytoolz.curry(__delitem__) +__mod__ = cytoolz.curry(__mod__) +and_ = cytoolz.curry(and_) +__iadd__ = cytoolz.curry(__iadd__) +isMappingType = cytoolz.curry(isMappingType) +setitem = cytoolz.curry(setitem) +__le__ = cytoolz.curry(__le__) __floordiv__ = cytoolz.curry(__floordiv__) -__ior__ = cytoolz.curry(__ior__) +__sub__ = cytoolz.curry(__sub__) +div = cytoolz.curry(div) +__ge__ = cytoolz.curry(__ge__) +ifloordiv = cytoolz.curry(ifloordiv) __ipow__ = cytoolz.curry(__ipow__) -__xor__ = cytoolz.curry(__xor__) +ixor = cytoolz.curry(ixor) +__isub__ = cytoolz.curry(__isub__) +__and__ = cytoolz.curry(__and__) __truediv__ = cytoolz.curry(__truediv__) -_abs = cytoolz.curry(_abs) -ifloordiv = cytoolz.curry(ifloordiv) -attrgetter = cytoolz.curry(attrgetter) +repeat = cytoolz.curry(repeat) +__repeat__ = cytoolz.curry(__repeat__) +__imod__ = cytoolz.curry(__imod__) +eq = cytoolz.curry(eq) +iadd = cytoolz.curry(iadd) +__delslice__ = cytoolz.curry(__delslice__) +xor = cytoolz.curry(xor) +sub = cytoolz.curry(sub) __contains__ = cytoolz.curry(__contains__) -__abs__ = cytoolz.curry(__abs__) -__not__ = cytoolz.curry(__not__) -__iconcat__ = cytoolz.curry(__iconcat__) -__ixor__ = cytoolz.curry(__ixor__) -__setitem__ = cytoolz.curry(__setitem__) -lshift = cytoolz.curry(lshift) -mod = cytoolz.curry(mod) -le = cytoolz.curry(le) +getslice = cytoolz.curry(getslice) +ne = cytoolz.curry(ne) +idiv = cytoolz.curry(idiv) +indexOf = cytoolz.curry(indexOf) +attrgetter = cytoolz.curry(attrgetter) +__invert__ = cytoolz.curry(__invert__) +methodcaller = cytoolz.curry(methodcaller) +truediv = cytoolz.curry(truediv) +mul = cytoolz.curry(mul) __neg__ = cytoolz.curry(__neg__) -isub = cytoolz.curry(isub) -is_not = cytoolz.curry(is_not) +__ne__ = cytoolz.curry(__ne__) +irepeat = cytoolz.curry(irepeat) +is_ = cytoolz.curry(is_) +getitem = cytoolz.curry(getitem) __ifloordiv__ = cytoolz.curry(__ifloordiv__) -or_ = cytoolz.curry(or_) -__concat__ = cytoolz.curry(__concat__) -__le__ = cytoolz.curry(__le__) -lt = cytoolz.curry(lt) +__idiv__ = cytoolz.curry(__idiv__) +isNumberType = cytoolz.curry(isNumberType) +ior = cytoolz.curry(ior) +__setitem__ = cytoolz.curry(__setitem__) +__or__ = cytoolz.curry(__or__) __add__ = cytoolz.curry(__add__) -__iand__ = cytoolz.curry(__iand__) -__sub__ = cytoolz.curry(__sub__) -__gt__ = cytoolz.curry(__gt__) -__mod__ = cytoolz.curry(__mod__) -ne = cytoolz.curry(ne) +ilshift = cytoolz.curry(ilshift) +__ilshift__ = cytoolz.curry(__ilshift__) +or_ = cytoolz.curry(or_) __imul__ = cytoolz.curry(__imul__) +is_not = cytoolz.curry(is_not) +__not__ = cytoolz.curry(__not__) +__setslice__ = cytoolz.curry(__setslice__) +imod = cytoolz.curry(imod) +itruediv = cytoolz.curry(itruediv) +__xor__ = cytoolz.curry(__xor__) +__ior__ = cytoolz.curry(__ior__) +__div__ = cytoolz.curry(__div__) +floordiv = cytoolz.curry(floordiv) +__mul__ = cytoolz.curry(__mul__) +__index__ = cytoolz.curry(__index__) itemgetter = cytoolz.curry(itemgetter) -__or__ = cytoolz.curry(__or__) -ixor = cytoolz.curry(ixor) -__isub__ = cytoolz.curry(__isub__) -is_ = cytoolz.curry(is_) +delitem = cytoolz.curry(delitem) del cytoolz diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index c4969e6..216c6b7 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -20,6 +20,9 @@ __all__ = ['identity', 'thread_first', 'thread_last', 'memoize', 'compose', 'pipe', 'complement', 'juxt', 'do', 'curry', 'memoize'] +PY2 = sys.version_info[0] == 2 + + cpdef object identity(object x): return x @@ -204,7 +207,7 @@ cdef class curry: self.func = func self.args = args - self.keywords = kwargs if kwargs else None + self.keywords = kwargs if kwargs or PY2 else None self.__doc__ = getattr(func, '__doc__', None) self.__name__ = getattr(func, '__name__', '') diff --git a/etc/generate_curried.py b/etc/generate_curried.py index 0b3c4ce..7459e46 100644 --- a/etc/generate_curried.py +++ b/etc/generate_curried.py @@ -10,10 +10,9 @@ Please make all edits in etc/generate_curried.py and re-run cytoolz/tests/test_curried_toolzlike.py """ - import cytoolz from cytoolz import * -from .operator import * +from . import operator {curried} @@ -42,6 +41,7 @@ def gen_init(): Please make all edits in etc/generate_curried.py and re-run cytoolz/tests/test_curried_toolzlike.py """ +from __future__ import absolute_import from operator import ( {imports} @@ -56,8 +56,12 @@ def gen_init(): def gen_op(): + special = set([ + 'absolute_import', + '__builtins__', + ]) return op_template.format( - imports='\n '.join(k + ',' for k in vars(cop)), + imports='\n '.join(k + ',' for k in vars(cop) if k not in special), curried='\n'.join( '{0} = cytoolz.curry({0})'.format(k) for k, v in vars(cop).items() if isinstance(v, toolz.curry) From 9b7a6c22b91ce205f605af5046f4d3bf6fb1547f Mon Sep 17 00:00:00 2001 From: llllllllll Date: Sat, 1 Aug 2015 16:57:24 -0400 Subject: [PATCH 072/146] MAINT: update curry --- cytoolz/curried/__init__.py | 143 +++++++++++++------- cytoolz/curried/operator.py | 258 ++++-------------------------------- etc/generate_curried.py | 123 +++++++++-------- 3 files changed, 189 insertions(+), 335 deletions(-) mode change 100644 => 100755 etc/generate_curried.py diff --git a/cytoolz/curried/__init__.py b/cytoolz/curried/__init__.py index 03c6581..1220b72 100644 --- a/cytoolz/curried/__init__.py +++ b/cytoolz/curried/__init__.py @@ -1,57 +1,102 @@ +############################################################################### +# THIS FILE IS AUTOMATICALLY GENERATED # +# # +# Please make all edits in etc/generate_curried.py and re-run # +# cytoolz/tests/test_curried_toolzlike.py # +############################################################################### """ -THIS FILE IS AUTOMATICALLY GENERATED +Alternate namespece for cytoolz such that all functions are curried -Please make all edits in etc/generate_curried.py and re-run -cytoolz/tests/test_curried_toolzlike.py +Currying provides implicit partial evaluation of all functions + +Example: + + Get usually requires two arguments, an index and a collection + >>> from cytoolz.curried import get + >>> get(0, ('a', 'b')) + 'a' + + When we use it in higher order functions we often want to pass a partially + evaluated form + >>> data = [(1, 2), (11, 22), (111, 222)] + >>> list(map(lambda seq: get(0, seq), data)) + [1, 11, 111] + + The curried version allows simple expression of partial evaluation + >>> list(map(get(0), data)) + [1, 11, 111] + +See Also: + cytoolz.functoolz.curry """ -import cytoolz -from cytoolz import * +import inspect + +from . import exceptions from . import operator +import cytoolz + + +def _should_curry(f): + return f in set([ + cytoolz.partition, + cytoolz.unique, + cytoolz.take, + cytoolz.map, + cytoolz.topk, + cytoolz.join, + cytoolz.interleave, + cytoolz.assoc, + cytoolz.countby, + cytoolz.pluck, + cytoolz.sliding_window, + cytoolz.groupby, + cytoolz.reduceby, + cytoolz.dissoc, + cytoolz.keymap, + cytoolz.merge_with, + cytoolz.itemfilter, + cytoolz.memoize, + cytoolz.iterate, + cytoolz.accumulate, + cytoolz.valmap, + cytoolz.cons, + cytoolz.do, + cytoolz.sorted, + cytoolz.get_in, + cytoolz.remove, + cytoolz.mapcat, + cytoolz.take_nth, + cytoolz.get, + cytoolz.interpose, + cytoolz.itemmap, + cytoolz.nth, + cytoolz.partitionby, + cytoolz.drop, + cytoolz.merge, + cytoolz.reduce, + cytoolz.filter, + cytoolz.update_in, + cytoolz.keyfilter, + cytoolz.tail, + cytoolz.valfilter, + cytoolz.partition_all, + ]) + + +def _curry_namespace(ns): + return dict( + (name, cytoolz.curry(f) if _should_curry(f) else f) + for name, f in ns.items() if '__' not in name + ) + +locals().update(cytoolz.merge( + _curry_namespace(vars(cytoolz)), + _curry_namespace(vars(exceptions)), +)) -reduceby = cytoolz.curry(reduceby) -itemmap = cytoolz.curry(itemmap) -iterate = cytoolz.curry(iterate) -update_in = cytoolz.curry(update_in) -dissoc = cytoolz.curry(dissoc) -tail = cytoolz.curry(tail) -do = cytoolz.curry(do) -merge_with = cytoolz.curry(merge_with) -get = cytoolz.curry(get) -itemfilter = cytoolz.curry(itemfilter) -accumulate = cytoolz.curry(accumulate) -sorted = cytoolz.curry(sorted) -unique = cytoolz.curry(unique) -join = cytoolz.curry(join) -drop = cytoolz.curry(drop) -remove = cytoolz.curry(remove) -interleave = cytoolz.curry(interleave) -assoc = cytoolz.curry(assoc) -cons = cytoolz.curry(cons) -map = cytoolz.curry(map) -reduce = cytoolz.curry(reduce) -mapcat = cytoolz.curry(mapcat) -countby = cytoolz.curry(countby) -take_nth = cytoolz.curry(take_nth) -merge = cytoolz.curry(merge) -keyfilter = cytoolz.curry(keyfilter) -take = cytoolz.curry(take) -memoize = cytoolz.curry(memoize) -partitionby = cytoolz.curry(partitionby) -get_in = cytoolz.curry(get_in) -valmap = cytoolz.curry(valmap) -interpose = cytoolz.curry(interpose) -pluck = cytoolz.curry(pluck) -nth = cytoolz.curry(nth) -topk = cytoolz.curry(topk) -sliding_window = cytoolz.curry(sliding_window) -partition = cytoolz.curry(partition) -filter = cytoolz.curry(filter) -keymap = cytoolz.curry(keymap) -partition_all = cytoolz.curry(partition_all) -valfilter = cytoolz.curry(valfilter) -groupby = cytoolz.curry(groupby) - -from .exceptions import * +# Clean up the namespace. +del _should_curry +del exceptions del cytoolz diff --git a/cytoolz/curried/operator.py b/cytoolz/curried/operator.py index c65b61b..0eda560 100644 --- a/cytoolz/curried/operator.py +++ b/cytoolz/curried/operator.py @@ -1,240 +1,30 @@ -""" -THIS FILE IS AUTOMATICALLY GENERATED - -Please make all edits in etc/generate_curried.py and re-run -cytoolz/tests/test_curried_toolzlike.py -""" from __future__ import absolute_import -from operator import ( - iand, - countOf, - gt, - __lshift__, - __concat__, - pow, - imul, - __iand__, - ge, - __irshift__, - setslice, - lshift, - ipow, - __lt__, - sequenceIncludes, - __inv__, - __rshift__, - le, - __getslice__, - __pos__, - invert, - contains, - lt, - __abs__, - isSequenceType, - add, - delslice, - _compare_digest, - concat, - irshift, - __iconcat__, - __ixor__, - __doc__, - __itruediv__, - rshift, - __getitem__, - isub, - __file__, - pos, - __irepeat__, - __pow__, - iconcat, - __gt__, - isCallable, - __eq__, - mod, - __delitem__, - __mod__, - and_, - __iadd__, - isMappingType, - setitem, - __le__, - truth, - __floordiv__, - __sub__, - div, - __ge__, - ifloordiv, - __ipow__, - ixor, - __isub__, - inv, - not_, - __and__, - __truediv__, - repeat, - __repeat__, - __imod__, - eq, - iadd, - index, - __delslice__, - xor, - sub, - __contains__, - neg, - getslice, - ne, - idiv, - __package__, - indexOf, - attrgetter, - abs, - __invert__, - methodcaller, - truediv, - mul, - __neg__, - __ne__, - irepeat, - is_, - getitem, - __ifloordiv__, - __idiv__, - isNumberType, - ior, - __setitem__, - __or__, - __add__, - __name__, - ilshift, - __ilshift__, - or_, - __imul__, - is_not, - __not__, - __setslice__, - imod, - itruediv, - __xor__, - __ior__, - __div__, - floordiv, - __mul__, - __index__, - itemgetter, - delitem, -) +import operator -import cytoolz +from cytoolz import curry -iand = cytoolz.curry(iand) -countOf = cytoolz.curry(countOf) -gt = cytoolz.curry(gt) -__lshift__ = cytoolz.curry(__lshift__) -__concat__ = cytoolz.curry(__concat__) -pow = cytoolz.curry(pow) -imul = cytoolz.curry(imul) -__iand__ = cytoolz.curry(__iand__) -ge = cytoolz.curry(ge) -__irshift__ = cytoolz.curry(__irshift__) -setslice = cytoolz.curry(setslice) -lshift = cytoolz.curry(lshift) -ipow = cytoolz.curry(ipow) -__lt__ = cytoolz.curry(__lt__) -sequenceIncludes = cytoolz.curry(sequenceIncludes) -__inv__ = cytoolz.curry(__inv__) -__rshift__ = cytoolz.curry(__rshift__) -le = cytoolz.curry(le) -__getslice__ = cytoolz.curry(__getslice__) -__pos__ = cytoolz.curry(__pos__) -contains = cytoolz.curry(contains) -lt = cytoolz.curry(lt) -__abs__ = cytoolz.curry(__abs__) -isSequenceType = cytoolz.curry(isSequenceType) -add = cytoolz.curry(add) -delslice = cytoolz.curry(delslice) -_compare_digest = cytoolz.curry(_compare_digest) -concat = cytoolz.curry(concat) -irshift = cytoolz.curry(irshift) -__iconcat__ = cytoolz.curry(__iconcat__) -__ixor__ = cytoolz.curry(__ixor__) -__itruediv__ = cytoolz.curry(__itruediv__) -rshift = cytoolz.curry(rshift) -__getitem__ = cytoolz.curry(__getitem__) -isub = cytoolz.curry(isub) -__irepeat__ = cytoolz.curry(__irepeat__) -__pow__ = cytoolz.curry(__pow__) -iconcat = cytoolz.curry(iconcat) -__gt__ = cytoolz.curry(__gt__) -isCallable = cytoolz.curry(isCallable) -__eq__ = cytoolz.curry(__eq__) -mod = cytoolz.curry(mod) -__delitem__ = cytoolz.curry(__delitem__) -__mod__ = cytoolz.curry(__mod__) -and_ = cytoolz.curry(and_) -__iadd__ = cytoolz.curry(__iadd__) -isMappingType = cytoolz.curry(isMappingType) -setitem = cytoolz.curry(setitem) -__le__ = cytoolz.curry(__le__) -__floordiv__ = cytoolz.curry(__floordiv__) -__sub__ = cytoolz.curry(__sub__) -div = cytoolz.curry(div) -__ge__ = cytoolz.curry(__ge__) -ifloordiv = cytoolz.curry(ifloordiv) -__ipow__ = cytoolz.curry(__ipow__) -ixor = cytoolz.curry(ixor) -__isub__ = cytoolz.curry(__isub__) -__and__ = cytoolz.curry(__and__) -__truediv__ = cytoolz.curry(__truediv__) -repeat = cytoolz.curry(repeat) -__repeat__ = cytoolz.curry(__repeat__) -__imod__ = cytoolz.curry(__imod__) -eq = cytoolz.curry(eq) -iadd = cytoolz.curry(iadd) -__delslice__ = cytoolz.curry(__delslice__) -xor = cytoolz.curry(xor) -sub = cytoolz.curry(sub) -__contains__ = cytoolz.curry(__contains__) -getslice = cytoolz.curry(getslice) -ne = cytoolz.curry(ne) -idiv = cytoolz.curry(idiv) -indexOf = cytoolz.curry(indexOf) -attrgetter = cytoolz.curry(attrgetter) -__invert__ = cytoolz.curry(__invert__) -methodcaller = cytoolz.curry(methodcaller) -truediv = cytoolz.curry(truediv) -mul = cytoolz.curry(mul) -__neg__ = cytoolz.curry(__neg__) -__ne__ = cytoolz.curry(__ne__) -irepeat = cytoolz.curry(irepeat) -is_ = cytoolz.curry(is_) -getitem = cytoolz.curry(getitem) -__ifloordiv__ = cytoolz.curry(__ifloordiv__) -__idiv__ = cytoolz.curry(__idiv__) -isNumberType = cytoolz.curry(isNumberType) -ior = cytoolz.curry(ior) -__setitem__ = cytoolz.curry(__setitem__) -__or__ = cytoolz.curry(__or__) -__add__ = cytoolz.curry(__add__) -ilshift = cytoolz.curry(ilshift) -__ilshift__ = cytoolz.curry(__ilshift__) -or_ = cytoolz.curry(or_) -__imul__ = cytoolz.curry(__imul__) -is_not = cytoolz.curry(is_not) -__not__ = cytoolz.curry(__not__) -__setslice__ = cytoolz.curry(__setslice__) -imod = cytoolz.curry(imod) -itruediv = cytoolz.curry(itruediv) -__xor__ = cytoolz.curry(__xor__) -__ior__ = cytoolz.curry(__ior__) -__div__ = cytoolz.curry(__div__) -floordiv = cytoolz.curry(floordiv) -__mul__ = cytoolz.curry(__mul__) -__index__ = cytoolz.curry(__index__) -itemgetter = cytoolz.curry(itemgetter) -delitem = cytoolz.curry(delitem) -del cytoolz +# We use a blacklist instead of whitelist because: +# 1. We have more things to include than exclude. +# 2. This gives us access to things like matmul iff we are in Python >=3.5. +no_curry = frozenset(( + 'abs', + 'index', + 'inv', + 'invert', + 'neg', + 'not_', + 'pos', + 'truth', +)) + +locals().update( + dict((name, curry(f) if name not in no_curry else f) + for name, f in vars(operator).items() if callable(f)), +) +# Clean up the namespace. +del curry +del no_curry +del operator diff --git a/etc/generate_curried.py b/etc/generate_curried.py old mode 100644 new mode 100755 index 7459e46..ad48b10 --- a/etc/generate_curried.py +++ b/etc/generate_curried.py @@ -1,82 +1,101 @@ +#!/usr/bin/env python +from itertools import chain +import inspect + import toolz import toolz.curried -from toolz.curried import operator as cop -init_template = '''\ +autogen_header = """\ +############################################################################### +# THIS FILE IS AUTOMATICALLY GENERATED # +# # +# Please make all edits in etc/generate_curried.py and re-run # +# cytoolz/tests/test_curried_toolzlike.py # +############################################################################### """ -THIS FILE IS AUTOMATICALLY GENERATED -Please make all edits in etc/generate_curried.py and re-run -cytoolz/tests/test_curried_toolzlike.py + +init_template = '''\ """ -import cytoolz -from cytoolz import * -from . import operator +Alternate namespece for cytoolz such that all functions are curried +Currying provides implicit partial evaluation of all functions -{curried} +Example: -from .exceptions import * -del cytoolz -''' + Get usually requires two arguments, an index and a collection + >>> from cytoolz.curried import get + >>> get(0, ('a', 'b')) + 'a' + When we use it in higher order functions we often want to pass a partially + evaluated form + >>> data = [(1, 2), (11, 22), (111, 222)] + >>> list(map(lambda seq: get(0, seq), data)) + [1, 11, 111] -def gen_init(): - tc = tuple( - (k, v) for k, v in vars(toolz.curried).items() - if isinstance(v, toolz.curry) and 'operator' not in v.func.__module__ - ) - return init_template.format( - curried='\n'.join( - '{0} = cytoolz.curry({0})'.format(k) for k, v in tc - ), - ) + The curried version allows simple expression of partial evaluation + >>> list(map(get(0), data)) + [1, 11, 111] - -op_template = '''\ +See Also: + cytoolz.functoolz.curry """ -THIS FILE IS AUTOMATICALLY GENERATED +import inspect -Please make all edits in etc/generate_curried.py and re-run -cytoolz/tests/test_curried_toolzlike.py -""" -from __future__ import absolute_import +from . import exceptions +from . import operator +import cytoolz -from operator import ( - {imports} -) -import cytoolz +def _should_curry(f): + return f in set([ + {should_curry} + ]) + + +def _curry_namespace(ns): + return dict( + (name, cytoolz.curry(f) if _should_curry(f) else f) + for name, f in ns.items() if '__' not in name + ) + -{curried} +locals().update(cytoolz.merge( + _curry_namespace(vars(cytoolz)), + _curry_namespace(vars(exceptions)), +)) +# Clean up the namespace. +del _should_curry +del exceptions del cytoolz ''' -def gen_op(): - special = set([ - 'absolute_import', - '__builtins__', - ]) - return op_template.format( - imports='\n '.join(k + ',' for k in vars(cop) if k not in special), - curried='\n'.join( - '{0} = cytoolz.curry({0})'.format(k) - for k, v in vars(cop).items() if isinstance(v, toolz.curry) - ) +def _curry_namespace(ns): + return ( + 'cytoolz.' + name + ',' + for name, f in ns.items() if isinstance(f, toolz.curry) + ) + + + +def gen_init(): + return init_template.format( + should_curry='\n '.join( + _curry_namespace(vars(toolz.curried)), + ), ) def main(argv): - if len(argv) == 1 or argv[1].lower() == 'init': - fn = gen_init - elif argv[1].lower() in ('op', 'operator'): - fn = gen_op - else: - raise ValueError('%s is not a valid argument' % argv[1]) - print(fn()) + if len(argv) != 1: + raise ValueError('no arguments expected') + + + print(autogen_header + gen_init()) if __name__ == '__main__': From 030493f02adbcfd6c941a1d7bc6a1c2bd929a9ae Mon Sep 17 00:00:00 2001 From: llllllllll Date: Sat, 1 Aug 2015 18:09:20 -0400 Subject: [PATCH 073/146] BUG: 2.7.10 compat kwargs --- cytoolz/functoolz.pyx | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 216c6b7..a3c3c06 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -1,9 +1,10 @@ #cython: embedsignature=True import inspect import sys +from functools import partial from cytoolz.compatibility import filter as ifilter, map as imap, reduce -from cpython.dict cimport PyDict_Merge, PyDict_New +from cpython.dict cimport PyDict_Merge, PyDict_New, PyDict_Copy from cpython.exc cimport PyErr_Clear, PyErr_ExceptionMatches, PyErr_Occurred from cpython.object cimport (PyCallable_Check, PyObject_Call, PyObject_CallObject, PyObject_RichCompare, Py_EQ, Py_NE) @@ -20,9 +21,6 @@ __all__ = ['identity', 'thread_first', 'thread_last', 'memoize', 'compose', 'pipe', 'complement', 'juxt', 'do', 'curry', 'memoize'] -PY2 = sys.version_info[0] == 2 - - cpdef object identity(object x): return x @@ -154,6 +152,25 @@ cpdef Py_ssize_t _num_required_args(object func) except *: return -1 +cdef struct partialobject: + PyObject _ + PyObject *fn + PyObject *args + PyObject *kw + PyObject *dict + PyObject *weakreflist + + +cdef object _partial = partial(lambda: None) + + +cdef object _empty_kwargs(): + kwds = ( _partial).kw + if kwds == NULL or kwds is None: + return None + return PyDict_Copy( kwds) + + cdef class curry: """ curry(self, *args, **kwargs) @@ -207,7 +224,7 @@ cdef class curry: self.func = func self.args = args - self.keywords = kwargs if kwargs or PY2 else None + self.keywords = kwargs if kwargs else _empty_kwargs() self.__doc__ = getattr(func, '__doc__', None) self.__name__ = getattr(func, '__name__', '') From bee4d0180547f03a1b65f3a169b7bcb073da4382 Mon Sep 17 00:00:00 2001 From: llllllllll Date: Sat, 1 Aug 2015 18:12:35 -0400 Subject: [PATCH 074/146] ENH: cannot be null in fully initalized state --- cytoolz/functoolz.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index a3c3c06..b71084c 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -165,10 +165,10 @@ cdef object _partial = partial(lambda: None) cdef object _empty_kwargs(): - kwds = ( _partial).kw - if kwds == NULL or kwds is None: + kwds = ( _partial).kw + if kwds is None: return None - return PyDict_Copy( kwds) + return PyDict_Copy(kwds) cdef class curry: From 4fca904966362956ddbd166d28707eeadb3bec46 Mon Sep 17 00:00:00 2001 From: llllllllll Date: Sun, 2 Aug 2015 18:10:20 -0400 Subject: [PATCH 075/146] MAINT: makes the lookup faster --- cytoolz/curried/__init__.py | 95 ++++++++++++++++++------------------- cytoolz/functoolz.pyx | 7 ++- etc/generate_curried.py | 25 ++++------ 3 files changed, 59 insertions(+), 68 deletions(-) diff --git a/cytoolz/curried/__init__.py b/cytoolz/curried/__init__.py index 1220b72..d10ff09 100644 --- a/cytoolz/curried/__init__.py +++ b/cytoolz/curried/__init__.py @@ -29,63 +29,59 @@ See Also: cytoolz.functoolz.curry """ -import inspect - from . import exceptions from . import operator import cytoolz -def _should_curry(f): - return f in set([ - cytoolz.partition, - cytoolz.unique, - cytoolz.take, - cytoolz.map, - cytoolz.topk, - cytoolz.join, - cytoolz.interleave, - cytoolz.assoc, - cytoolz.countby, - cytoolz.pluck, - cytoolz.sliding_window, - cytoolz.groupby, - cytoolz.reduceby, - cytoolz.dissoc, - cytoolz.keymap, - cytoolz.merge_with, - cytoolz.itemfilter, - cytoolz.memoize, - cytoolz.iterate, - cytoolz.accumulate, - cytoolz.valmap, - cytoolz.cons, - cytoolz.do, - cytoolz.sorted, - cytoolz.get_in, - cytoolz.remove, - cytoolz.mapcat, - cytoolz.take_nth, - cytoolz.get, - cytoolz.interpose, - cytoolz.itemmap, - cytoolz.nth, - cytoolz.partitionby, - cytoolz.drop, - cytoolz.merge, - cytoolz.reduce, - cytoolz.filter, - cytoolz.update_in, - cytoolz.keyfilter, - cytoolz.tail, - cytoolz.valfilter, - cytoolz.partition_all, - ]) +_curry_set = frozenset([ + cytoolz.nth, + cytoolz.partition, + cytoolz.take_nth, + cytoolz.tail, + cytoolz.valfilter, + cytoolz.memoize, + cytoolz.reduceby, + cytoolz.topk, + cytoolz.join, + cytoolz.do, + cytoolz.sorted, + cytoolz.interpose, + cytoolz.take, + cytoolz.pluck, + cytoolz.drop, + cytoolz.get_in, + cytoolz.reduce, + cytoolz.itemfilter, + cytoolz.accumulate, + cytoolz.merge, + cytoolz.interleave, + cytoolz.iterate, + cytoolz.get, + cytoolz.remove, + cytoolz.valmap, + cytoolz.keymap, + cytoolz.cons, + cytoolz.unique, + cytoolz.partitionby, + cytoolz.itemmap, + cytoolz.sliding_window, + cytoolz.map, + cytoolz.partition_all, + cytoolz.assoc, + cytoolz.mapcat, + cytoolz.filter, + cytoolz.countby, + cytoolz.merge_with, + cytoolz.update_in, + cytoolz.keyfilter, + cytoolz.groupby, +]) def _curry_namespace(ns): return dict( - (name, cytoolz.curry(f) if _should_curry(f) else f) + (name, cytoolz.curry(f) if f in _curry_set else f) for name, f in ns.items() if '__' not in name ) @@ -96,7 +92,8 @@ def _curry_namespace(ns): )) # Clean up the namespace. -del _should_curry +del _curry_set +del _curry_namespace del exceptions del cytoolz diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index b71084c..a0eeb9d 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -4,7 +4,7 @@ import sys from functools import partial from cytoolz.compatibility import filter as ifilter, map as imap, reduce -from cpython.dict cimport PyDict_Merge, PyDict_New, PyDict_Copy +from cpython.dict cimport PyDict_Merge, PyDict_New from cpython.exc cimport PyErr_Clear, PyErr_ExceptionMatches, PyErr_Occurred from cpython.object cimport (PyCallable_Check, PyObject_Call, PyObject_CallObject, PyObject_RichCompare, Py_EQ, Py_NE) @@ -165,10 +165,9 @@ cdef object _partial = partial(lambda: None) cdef object _empty_kwargs(): - kwds = ( _partial).kw - if kwds is None: + if ( _partial).kw is None: return None - return PyDict_Copy(kwds) + return PyDict_New() cdef class curry: diff --git a/etc/generate_curried.py b/etc/generate_curried.py index ad48b10..e6114b2 100755 --- a/etc/generate_curried.py +++ b/etc/generate_curried.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from itertools import chain -import inspect - +import cytoolz import toolz import toolz.curried @@ -42,22 +40,19 @@ See Also: cytoolz.functoolz.curry """ -import inspect - from . import exceptions from . import operator import cytoolz -def _should_curry(f): - return f in set([ - {should_curry} - ]) +_curry_set = frozenset([ + {should_curry} +]) def _curry_namespace(ns): return dict( - (name, cytoolz.curry(f) if _should_curry(f) else f) + (name, cytoolz.curry(f) if f in _curry_set else f) for name, f in ns.items() if '__' not in name ) @@ -68,23 +63,24 @@ def _curry_namespace(ns): )) # Clean up the namespace. -del _should_curry +del _curry_set +del _curry_namespace del exceptions del cytoolz ''' def _curry_namespace(ns): + ct = vars(cytoolz) return ( 'cytoolz.' + name + ',' - for name, f in ns.items() if isinstance(f, toolz.curry) + for name, f in ns.items() if isinstance(f, toolz.curry) and name in ct ) - def gen_init(): return init_template.format( - should_curry='\n '.join( + should_curry='\n '.join( _curry_namespace(vars(toolz.curried)), ), ) @@ -94,7 +90,6 @@ def main(argv): if len(argv) != 1: raise ValueError('no arguments expected') - print(autogen_header + gen_init()) From 24f04997622292493ba863430b51e71a8bd768a1 Mon Sep 17 00:00:00 2001 From: llllllllll Date: Sun, 2 Aug 2015 18:25:12 -0400 Subject: [PATCH 076/146] ENH: Adds flip --- cytoolz/functoolz.pyx | 9 ++++++++- cytoolz/tests/test_functoolz.py | 10 ++++++++-- cytoolz/tests/test_none_safe.py | 3 +++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index c4969e6..3c824cc 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -17,7 +17,7 @@ from cytoolz.cpython cimport PtrObject_Call __all__ = ['identity', 'thread_first', 'thread_last', 'memoize', 'compose', - 'pipe', 'complement', 'juxt', 'do', 'curry', 'memoize'] + 'pipe', 'complement', 'juxt', 'do', 'curry', 'memoize', 'flip'] cpdef object identity(object x): @@ -580,3 +580,10 @@ cpdef object do(object func, object x): """ func(x) return x + + +cpdef object _flip(object f, object a, object b): + return PyObject_CallObject(f, (b, a)) + + +flip = curry(_flip) diff --git a/cytoolz/tests/test_functoolz.py b/cytoolz/tests/test_functoolz.py index acd8d37..79ad865 100644 --- a/cytoolz/tests/test_functoolz.py +++ b/cytoolz/tests/test_functoolz.py @@ -2,12 +2,11 @@ from cytoolz.functoolz import (thread_first, thread_last, memoize, curry, - compose, pipe, complement, do, juxt) + compose, pipe, complement, do, juxt, flip) from cytoolz.functoolz import _num_required_args from operator import add, mul, itemgetter from cytoolz.utils import raises from functools import partial -from cytoolz.compatibility import reduce, PY3 def iseven(x): @@ -484,3 +483,10 @@ def test_juxt_generator_input(): juxtfunc = juxt(itemgetter(2*i) for i in range(5)) assert juxtfunc(data) == (0, 2, 4, 6, 8) assert juxtfunc(data) == (0, 2, 4, 6, 8) + + +def test_flip(): + def f(a, b): + return a, b + + assert flip(f, 'a', 'b') == ('b', 'a') diff --git a/cytoolz/tests/test_none_safe.py b/cytoolz/tests/test_none_safe.py index f7ad5ec..4de217e 100644 --- a/cytoolz/tests/test_none_safe.py +++ b/cytoolz/tests/test_none_safe.py @@ -140,6 +140,9 @@ def test_functoolz(): assert thread_last(1, None) is None tested.append('thread_last') + assert flip(lambda a, b: (a, b))(None)(None) == (None, None) + tested.append('flip') + s1 = set(tested) s2 = set(cytoolz.functoolz.__all__) assert s1 == s2, '%s not tested for being None-safe' % ', '.join(s2 - s1) From 816f3fa2b6ea3c5d486ec3c8761e5f9ab01db2e8 Mon Sep 17 00:00:00 2001 From: llllllllll Date: Sun, 9 Aug 2015 21:03:36 -0400 Subject: [PATCH 077/146] MAINT: rename start -> initial --- cytoolz/itertoolz.pxd | 2 +- cytoolz/itertoolz.pyx | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index 67edd85..0431e73 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -7,7 +7,7 @@ cdef class accumulate: cdef object binop cdef object iter_seq cdef object result - cdef object start + cdef object initial cpdef dict groupby(object key, object seq) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 4497873..7dddf2a 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -62,9 +62,7 @@ cdef class remove: cdef class accumulate: - """ accumulate(binop, seq) - - Repeatedly apply binary function to a sequence, accumulating results + """ Repeatedly apply binary function to a sequence, accumulating results >>> from operator import add, mul >>> list(accumulate(add, [1, 2, 3, 4, 5])) @@ -79,22 +77,30 @@ cdef class accumulate: >>> sum = partial(reduce, add) >>> cumsum = partial(accumulate, add) + Accumulate also takes an optional argument that will be used as the first + value. This is similar to reduce. + + >>> list(accumulate(add, [1, 2, 3], -1)) + [-1, 0, 2, 5] + >>> list(accumulate(add, [], 1)) + [1] + See Also: itertools.accumulate : In standard itertools for Python 3.2+ """ - def __cinit__(self, object binop, object seq, object start=no_default): + def __cinit__(self, object binop, object seq, object initial=no_default): self.binop = binop self.iter_seq = iter(seq) self.result = self # sentinel - self.start = start + self.initial = initial def __iter__(self): return self def __next__(self): if self.result is self: - if self.start is not no_default: - self.result = self.start + if self.initial is not no_default: + self.result = self.initial else: self.result = next(self.iter_seq) else: From 8e521d8fce18dd27a729e4612ab2198b991659f8 Mon Sep 17 00:00:00 2001 From: llllllllll Date: Sun, 9 Aug 2015 21:23:44 -0400 Subject: [PATCH 078/146] MAINT: no fancy no_default --- cytoolz/utils.pyx | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/cytoolz/utils.pyx b/cytoolz/utils.pyx index 6297cdb..9c71b9d 100644 --- a/cytoolz/utils.pyx +++ b/cytoolz/utils.pyx @@ -16,21 +16,11 @@ def raises(err, lamda): return True -@object.__new__ -class no_default(object): - def __new__(self): - raise TypeError("cannot create 'no_default' instances") - - def __str__(self): - return '' - __repr__ = __str__ - - try: # Attempt to get the no_default sentinel object from toolz from toolz.utils import no_default except ImportError: - pass + no_default = '__no_default__' def include_dirs(): From 3b1b93df36e544055648acdf3e641daf08fb0fb0 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 20 Oct 2015 23:19:01 -0500 Subject: [PATCH 079/146] Update cytoolz to toolz 0.7.4. Release tests enabled. This adds `peek`, var-args to `dissoc`, and nice names and docstrings of composed functions. --- cytoolz/__init__.pxd | 4 ++-- cytoolz/_version.py | 4 ++-- cytoolz/dicttoolz.pxd | 2 +- cytoolz/dicttoolz.pyx | 21 +++++++++++++------- cytoolz/functoolz.pxd | 4 ++-- cytoolz/functoolz.pyx | 35 ++++++++++++++++++++++++++++++--- cytoolz/itertoolz.pxd | 3 +++ cytoolz/itertoolz.pyx | 26 ++++++++++++++++++++++-- cytoolz/tests/test_curried.py | 21 +++++++++++++++++++- cytoolz/tests/test_dicttoolz.py | 1 + cytoolz/tests/test_functoolz.py | 20 ++++++++++++++++++- cytoolz/tests/test_itertoolz.py | 11 ++++++++++- cytoolz/tests/test_none_safe.py | 3 +++ 13 files changed, 133 insertions(+), 22 deletions(-) diff --git a/cytoolz/__init__.pxd b/cytoolz/__init__.pxd index af7bf39..4146561 100644 --- a/cytoolz/__init__.pxd +++ b/cytoolz/__init__.pxd @@ -3,7 +3,7 @@ from cytoolz.itertoolz cimport ( frequencies, interleave, interpose, isdistinct, isiterable, iterate, last, mapcat, nth, partition, partition_all, pluck, reduceby, remove, rest, second, sliding_window, take, tail, take_nth, unique, join, - c_diff, topk) + c_diff, topk, peek) from cytoolz.functoolz cimport ( @@ -12,7 +12,7 @@ from cytoolz.functoolz cimport ( from cytoolz.dicttoolz cimport ( - assoc, c_merge, c_merge_with, dissoc, get_in, keyfilter, keymap, + assoc, c_merge, c_merge_with, c_dissoc, get_in, keyfilter, keymap, itemfilter, itemmap, update_in, valfilter, valmap) diff --git a/cytoolz/_version.py b/cytoolz/_version.py index 5816fa9..7d0d165 100644 --- a/cytoolz/_version.py +++ b/cytoolz/_version.py @@ -1,2 +1,2 @@ -__version__ = '0.7.4dev' -__toolz_version__ = '0.7.2' +__version__ = '0.7.4' +__toolz_version__ = '0.7.4' diff --git a/cytoolz/dicttoolz.pxd b/cytoolz/dicttoolz.pxd index c40c19f..4c99698 100644 --- a/cytoolz/dicttoolz.pxd +++ b/cytoolz/dicttoolz.pxd @@ -35,7 +35,7 @@ cpdef object itemfilter(object predicate, object d, object factory=*) cpdef object assoc(object d, object key, object value, object factory=*) -cpdef object dissoc(object d, object key) +cdef object c_dissoc(object d, object keys) cpdef object update_in(object d, object keys, object func, object default=*, object factory=*) diff --git a/cytoolz/dicttoolz.pyx b/cytoolz/dicttoolz.pyx index a10efff..38ad58d 100644 --- a/cytoolz/dicttoolz.pyx +++ b/cytoolz/dicttoolz.pyx @@ -374,20 +374,27 @@ cpdef object assoc(object d, object key, object value, object factory=dict): return rv -cpdef object dissoc(object d, object key): +cdef object c_dissoc(object d, object keys): + cdef object rv, key + rv = copy(d) + for key in keys: + del rv[key] + return rv + + +def dissoc(d, *keys): """ - Return a new dict with the given key removed. + Return a new dict with the given key(s) removed. - New dict has d[key] deleted. + New dict has d[key] deleted for each supplied key. Does not modify the initial dictionary. >>> dissoc({'x': 1, 'y': 2}, 'y') {'x': 1} + >>> dissoc({'x': 1, 'y': 2}, 'y', 'x') + {} """ - cdef object rv - rv = copy(d) - del rv[key] - return rv + return c_dissoc(d, keys) cpdef object update_in(object d, object keys, object func, object default=None, object factory=dict): diff --git a/cytoolz/functoolz.pxd b/cytoolz/functoolz.pxd index b15db7d..a0d9f13 100644 --- a/cytoolz/functoolz.pxd +++ b/cytoolz/functoolz.pxd @@ -26,8 +26,8 @@ cpdef object memoize(object func=*, object cache=*, object key=*) cdef class Compose: - cdef object firstfunc - cdef tuple funcs + cdef public object first + cdef public tuple funcs cdef object c_compose(object funcs) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 94b8d95..4e252b2 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -431,22 +431,51 @@ cdef class Compose: compose """ def __cinit__(self, *funcs): - self.firstfunc = funcs[-1] + self.first = funcs[-1] self.funcs = tuple(reversed(funcs[:-1])) def __call__(self, *args, **kwargs): cdef object func, ret - ret = PyObject_Call(self.firstfunc, args, kwargs) + ret = PyObject_Call(self.first, args, kwargs) for func in self.funcs: ret = func(ret) return ret def __reduce__(self): - return (Compose, (self.firstfunc,), self.funcs) + return (Compose, (self.first,), self.funcs) def __setstate__(self, state): self.funcs = state + property __name__: + def __get__(self): + try: + return '_of_'.join( + f.__name__ for f in reversed((self.first,) + self.funcs) + ) + except AttributeError: + return type(self).__name__ + + property __doc__: + def __get__(self): + def composed_doc(*fs): + """Generate a docstring for the composition of fs. + """ + if not fs: + # Argument name for the docstring. + return '*args, **kwargs' + + return '{f}({g})'.format(f=fs[0].__name__, g=composed_doc(*fs[1:])) + + try: + return ( + 'lambda *args, **kwargs: ' + + composed_doc(*reversed((self.first,) + self.funcs)) + ) + except AttributeError: + # One of our callables does not have a `__name__`, whatever. + return 'A composition of functions' + cdef object c_compose(object funcs): if not funcs: diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index 0431e73..9ddb1d6 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -255,3 +255,6 @@ cdef object c_diff(object seqs, object default=*, object key=*) cpdef object topk(Py_ssize_t k, object seq, object key=*) + + +cpdef object peek(object seq) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 7dddf2a..65d6e51 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -25,7 +25,7 @@ __all__ = ['remove', 'accumulate', 'groupby', 'merge_sorted', 'interleave', 'first', 'second', 'nth', 'last', 'get', 'concat', 'concatv', 'mapcat', 'cons', 'interpose', 'frequencies', 'reduceby', 'iterate', 'sliding_window', 'partition', 'partition_all', 'count', 'pluck', - 'join', 'tail', 'diff', 'topk'] + 'join', 'tail', 'diff', 'topk', 'peek'] concatv = chain @@ -62,7 +62,9 @@ cdef class remove: cdef class accumulate: - """ Repeatedly apply binary function to a sequence, accumulating results + """ accumulate(binop, seq, initial='__no__default__') + + Repeatedly apply binary function to a sequence, accumulating results >>> from operator import add, mul >>> list(accumulate(add, [1, 2, 3, 4, 5])) @@ -1664,3 +1666,23 @@ cpdef object topk(Py_ssize_t k, object seq, object key=None): pq.sort(reverse=True) k = 0 if key is None else 2 return tuple([item[k] for item in pq]) + + +cpdef object peek(object seq): + """ + Retrieve the next element of a sequence + + Returns the first element and an iterable equivalent to the original + sequence, still having the element retrieved. + + >>> seq = [0, 1, 2, 3, 4] + >>> first, seq = peek(seq) + >>> first + 0 + >>> list(seq) + [0, 1, 2, 3, 4] + + """ + iterator = iter(seq) + item = next(iterator) + return item, chain((item,), iterator) diff --git a/cytoolz/tests/test_curried.py b/cytoolz/tests/test_curried.py index 6bd424d..0665d46 100644 --- a/cytoolz/tests/test_curried.py +++ b/cytoolz/tests/test_curried.py @@ -1,7 +1,7 @@ import cytoolz import cytoolz.curried from cytoolz.curried import (take, first, second, sorted, merge_with, reduce, - merge) + merge, operator as cop) from collections import defaultdict from operator import add @@ -38,3 +38,22 @@ def test_reduce(): def test_module_name(): assert cytoolz.curried.__name__ == 'cytoolz.curried' + + +def test_curried_operator(): + for k, v in vars(cop).items(): + if not callable(v): + continue + + if not isinstance(v, cytoolz.curry): + try: + # Make sure it is unary + # We cannot use isunary because it might be defined in C. + v(1) + except TypeError: + raise AssertionError( + 'cytoolz.curried.operator.%s is not curried!' % k, + ) + + # Make sure this isn't totally empty. + assert len(set(vars(cop)) & set(['add', 'sub', 'mul'])) == 3 diff --git a/cytoolz/tests/test_dicttoolz.py b/cytoolz/tests/test_dicttoolz.py index 7d07f60..3c213fd 100644 --- a/cytoolz/tests/test_dicttoolz.py +++ b/cytoolz/tests/test_dicttoolz.py @@ -93,6 +93,7 @@ def test_dissoc(self): assert dissoc(D({"a": 1}), "a") == D({}) assert dissoc(D({"a": 1, "b": 2}), "a") == D({"b": 2}) assert dissoc(D({"a": 1, "b": 2}), "b") == D({"a": 1}) + assert dissoc(D({"a": 1, "b": 2}), "a", "b") == D({}) # Verify immutability: d = D({'x': 1}) diff --git a/cytoolz/tests/test_functoolz.py b/cytoolz/tests/test_functoolz.py index 79ad865..3e278fd 100644 --- a/cytoolz/tests/test_functoolz.py +++ b/cytoolz/tests/test_functoolz.py @@ -2,7 +2,7 @@ from cytoolz.functoolz import (thread_first, thread_last, memoize, curry, - compose, pipe, complement, do, juxt, flip) + compose, pipe, complement, do, juxt, flip) from cytoolz.functoolz import _num_required_args from operator import add, mul, itemgetter from cytoolz.utils import raises @@ -435,6 +435,24 @@ def f(a, b, c=10): assert compose(str, inc, f)(1, 2, c=3) == '10' + # Define two functions with different names + def f(a): + return a + + def g(a): + return a + + composed = compose(f, g) + assert composed.__name__ == 'f_of_g' + assert composed.__doc__ == 'lambda *args, **kwargs: f(g(*args, **kwargs))' + + # Create an object with no __name__. + h = object() + + composed = compose(f, h) + assert composed.__name__ == 'Compose' + assert composed.__doc__ == 'A composition of functions' + def test_pipe(): assert pipe(1, inc) == 2 diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index cf1895c..84ca99f 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -11,7 +11,7 @@ reduceby, iterate, accumulate, sliding_window, count, partition, partition_all, take_nth, pluck, join, - diff, topk) + diff, topk, peek) from cytoolz.compatibility import range, filter from operator import add, mul @@ -465,3 +465,12 @@ def test_topk(): def test_topk_is_stable(): assert topk(4, [5, 9, 2, 1, 5, 3], key=lambda x: 1) == (5, 9, 2, 1) + + +def test_peek(): + alist = ["Alice", "Bob", "Carol"] + element, blist = peek(alist) + element == alist[0] + assert list(blist) == alist + + assert raises(StopIteration, lambda: peek([])) diff --git a/cytoolz/tests/test_none_safe.py b/cytoolz/tests/test_none_safe.py index 4de217e..62f6280 100644 --- a/cytoolz/tests/test_none_safe.py +++ b/cytoolz/tests/test_none_safe.py @@ -291,6 +291,9 @@ def test_itertoolz(): assert raises(TypeError, lambda: list(diff([None, None]))) tested.append('diff') + assert raises(TypeError, lambda: peek(None)) + tested.append('peek') + s1 = set(tested) s2 = set(cytoolz.itertoolz.__all__) assert s1 == s2, '%s not tested for being None-safe' % ', '.join(s2 - s1) From a456c0caeeaa6bde0ed1ff191b867c83cd1d2039 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 20 Oct 2015 23:31:39 -0500 Subject: [PATCH 080/146] update AUTHORS.md --- AUTHORS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AUTHORS.md b/AUTHORS.md index 76cb1d2..15cd4f8 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -9,3 +9,5 @@ Lars Buitinck [@larsmans](http://github.com/la scoder [@scoder](https://github.com/scoder/) Phillip Cloud [@cpcloud](https://github.com/cpcloud) + +Joe Jevnik [@llllllllll](https://github.com/llllllllll) From a1e77398679ff69e2d984960fc86aed8b55ac917 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 20 Oct 2015 23:37:19 -0500 Subject: [PATCH 081/146] add forgotten tests. --- cytoolz/tests/test_dicttoolz_class.py | 237 +++++++++++++++++++++++++ cytoolz/tests/test_dicttoolz_custom.py | 198 +++++++++++++++++++++ cytoolz/tests/test_dicttoolz_dict.py | 134 ++++++++++++++ 3 files changed, 569 insertions(+) create mode 100644 cytoolz/tests/test_dicttoolz_class.py create mode 100644 cytoolz/tests/test_dicttoolz_custom.py create mode 100644 cytoolz/tests/test_dicttoolz_dict.py diff --git a/cytoolz/tests/test_dicttoolz_class.py b/cytoolz/tests/test_dicttoolz_class.py new file mode 100644 index 0000000..7d07f60 --- /dev/null +++ b/cytoolz/tests/test_dicttoolz_class.py @@ -0,0 +1,237 @@ +from collections import defaultdict as _defaultdict +from cytoolz.dicttoolz import (merge, merge_with, valmap, keymap, update_in, + assoc, dissoc, keyfilter, valfilter, itemmap, + itemfilter) +from cytoolz.utils import raises +from cytoolz.compatibility import PY3 + + +def inc(x): + return x + 1 + + +def iseven(i): + return i % 2 == 0 + + +class TestDict(object): + """Test typical usage: dict inputs, no factory keyword. + + Class attributes: + D: callable that inputs a dict and creates or returns a MutableMapping + kw: kwargs dict to specify "factory" keyword (if applicable) + """ + D = dict + kw = {} + + def test_merge(self): + D, kw = self.D, self.kw + assert merge(D({1: 1, 2: 2}), D({3: 4}), **kw) == D({1: 1, 2: 2, 3: 4}) + + def test_merge_iterable_arg(self): + D, kw = self.D, self.kw + assert merge([D({1: 1, 2: 2}), D({3: 4})], **kw) == D({1: 1, 2: 2, 3: 4}) + + def test_merge_with(self): + D, kw = self.D, self.kw + dicts = D({1: 1, 2: 2}), D({1: 10, 2: 20}) + assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22}) + assert merge_with(tuple, *dicts, **kw) == D({1: (1, 10), 2: (2, 20)}) + + dicts = D({1: 1, 2: 2, 3: 3}), D({1: 10, 2: 20}) + assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22, 3: 3}) + assert merge_with(tuple, *dicts, **kw) == D({1: (1, 10), 2: (2, 20), 3: (3,)}) + + assert not merge_with(sum) + + def test_merge_with_iterable_arg(self): + D, kw = self.D, self.kw + dicts = D({1: 1, 2: 2}), D({1: 10, 2: 20}) + assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22}) + assert merge_with(sum, dicts, **kw) == D({1: 11, 2: 22}) + assert merge_with(sum, iter(dicts), **kw) == D({1: 11, 2: 22}) + + def test_valmap(self): + D, kw = self.D, self.kw + assert valmap(inc, D({1: 1, 2: 2}), **kw) == D({1: 2, 2: 3}) + + def test_keymap(self): + D, kw = self.D, self.kw + assert keymap(inc, D({1: 1, 2: 2}), **kw) == D({2: 1, 3: 2}) + + def test_itemmap(self): + D, kw = self.D, self.kw + assert itemmap(reversed, D({1: 2, 2: 4}), **kw) == D({2: 1, 4: 2}) + + def test_valfilter(self): + D, kw = self.D, self.kw + assert valfilter(iseven, D({1: 2, 2: 3}), **kw) == D({1: 2}) + + def test_keyfilter(self): + D, kw = self.D, self.kw + assert keyfilter(iseven, D({1: 2, 2: 3}), **kw) == D({2: 3}) + + def test_itemfilter(self): + D, kw = self.D, self.kw + assert itemfilter(lambda item: iseven(item[0]), D({1: 2, 2: 3}), **kw) == D({2: 3}) + assert itemfilter(lambda item: iseven(item[1]), D({1: 2, 2: 3}), **kw) == D({1: 2}) + + def test_assoc(self): + D, kw = self.D, self.kw + assert assoc(D({}), "a", 1, **kw) == D({"a": 1}) + assert assoc(D({"a": 1}), "a", 3, **kw) == D({"a": 3}) + assert assoc(D({"a": 1}), "b", 3, **kw) == D({"a": 1, "b": 3}) + + # Verify immutability: + d = D({'x': 1}) + oldd = d + assoc(d, 'x', 2, **kw) + assert d is oldd + + def test_dissoc(self): + D, kw = self.D, self.kw + assert dissoc(D({"a": 1}), "a") == D({}) + assert dissoc(D({"a": 1, "b": 2}), "a") == D({"b": 2}) + assert dissoc(D({"a": 1, "b": 2}), "b") == D({"a": 1}) + + # Verify immutability: + d = D({'x': 1}) + oldd = d + d2 = dissoc(d, 'x') + assert d is oldd + assert d2 is not oldd + + def test_update_in(self): + D, kw = self.D, self.kw + assert update_in(D({"a": 0}), ["a"], inc, **kw) == D({"a": 1}) + assert update_in(D({"a": 0, "b": 1}), ["b"], str, **kw) == D({"a": 0, "b": "1"}) + assert (update_in(D({"t": 1, "v": D({"a": 0})}), ["v", "a"], inc, **kw) == + D({"t": 1, "v": D({"a": 1})})) + # Handle one missing key. + assert update_in(D({}), ["z"], str, None, **kw) == D({"z": "None"}) + assert update_in(D({}), ["z"], inc, 0, **kw) == D({"z": 1}) + assert update_in(D({}), ["z"], lambda x: x+"ar", default="b", **kw) == D({"z": "bar"}) + # Same semantics as Clojure for multiple missing keys, ie. recursively + # create nested empty dictionaries to the depth specified by the + # keys with the innermost value set to f(default). + assert update_in(D({}), [0, 1], inc, default=-1, **kw) == D({0: D({1: 0})}) + assert update_in(D({}), [0, 1], str, default=100, **kw) == D({0: D({1: "100"})}) + assert (update_in(D({"foo": "bar", 1: 50}), ["d", 1, 0], str, 20, **kw) == + D({"foo": "bar", 1: 50, "d": D({1: D({0: "20"})})})) + # Verify immutability: + d = D({'x': 1}) + oldd = d + update_in(d, ['x'], inc, **kw) + assert d is oldd + + def test_factory(self): + D, kw = self.D, self.kw + assert merge(defaultdict(int, D({1: 2})), D({2: 3})) == {1: 2, 2: 3} + assert (merge(defaultdict(int, D({1: 2})), D({2: 3}), + factory=lambda: defaultdict(int)) == + defaultdict(int, D({1: 2, 2: 3}))) + assert not (merge(defaultdict(int, D({1: 2})), D({2: 3}), + factory=lambda: defaultdict(int)) == {1: 2, 2: 3}) + assert raises(TypeError, lambda: merge(D({1: 2}), D({2: 3}), factoryy=dict)) + + +class defaultdict(_defaultdict): + def __eq__(self, other): + return (super(defaultdict, self).__eq__(other) and + isinstance(other, _defaultdict) and + self.default_factory == other.default_factory) + + +class TestDefaultDict(TestDict): + """Test defaultdict as input and factory + + Class attributes: + D: callable that inputs a dict and creates or returns a MutableMapping + kw: kwargs dict to specify "factory" keyword (if applicable) + """ + @staticmethod + def D(dict_): + return defaultdict(int, dict_) + + kw = {'factory': lambda: defaultdict(int)} + + +class CustomMapping(object): + """Define methods of the MutableMapping protocol required by dicttoolz""" + def __init__(self, *args, **kwargs): + self._d = dict(*args, **kwargs) + + def __getitem__(self, key): + return self._d[key] + + def __setitem__(self, key, val): + self._d[key] = val + + def __delitem__(self, key): + del self._d[key] + + def __iter__(self): + return iter(self._d) + + def __len__(self): + return len(self._d) + + def __contains__(self, key): + return key in self._d + + def __eq__(self, other): + return isinstance(other, CustomMapping) and self._d == other._d + + def __ne__(self, other): + return not isinstance(other, CustomMapping) or self._d != other._d + + def keys(self): + return self._d.keys() + + def values(self): + return self._d.values() + + def items(self): + return self._d.items() + + def update(self, *args, **kwargs): + self._d.update(*args, **kwargs) + + # Should we require these to be defined for Python 2? + if not PY3: + def iterkeys(self): + return self._d.iterkeys() + + def itervalues(self): + return self._d.itervalues() + + def iteritems(self): + return self._d.iteritems() + + # Unused methods that are part of the MutableMapping protocol + #def get(self, key, *args): + # return self._d.get(key, *args) + + #def pop(self, key, *args): + # return self._d.pop(key, *args) + + #def popitem(self, key): + # return self._d.popitem() + + #def clear(self): + # self._d.clear() + + #def setdefault(self, key, *args): + # return self._d.setdefault(self, key, *args) + + +class TestCustomMapping(TestDict): + """Test CustomMapping as input and factory + + Class attributes: + D: callable that inputs a dict and creates or returns a MutableMapping + kw: kwargs dict to specify "factory" keyword (if applicable) + """ + D = CustomMapping + kw = {'factory': lambda: CustomMapping()} + diff --git a/cytoolz/tests/test_dicttoolz_custom.py b/cytoolz/tests/test_dicttoolz_custom.py new file mode 100644 index 0000000..0cfe123 --- /dev/null +++ b/cytoolz/tests/test_dicttoolz_custom.py @@ -0,0 +1,198 @@ +from collections import defaultdict as _defaultdict +from cytoolz.dicttoolz import (merge, merge_with, valmap, keymap, update_in, + assoc, dissoc, keyfilter, valfilter, itemmap, + itemfilter) +from cytoolz.utils import raises + + +class defaultdict(_defaultdict): + def __eq__(self, other): + return (super(defaultdict, self).__eq__(other) and + isinstance(other, _defaultdict) and + self.default_factory == other.default_factory) + + +class D(object): + def __init__(self, *args, **kwargs): + self._d = dict(*args, **kwargs) + + def __getitem__(self, key): + return self._d[key] + + def __setitem__(self, key, val): + self._d[key] = val + + def __delitem__(self, key): + del self._d[key] + + def __iter__(self): + return iter(self._d) + + def __len__(self): + return len(self._d) + + def __contains__(self, key): + return key in self._d + + def __eq__(self, other): + return isinstance(other, D) and self._d == other._d + + def __ne__(self, other): + return not isinstance(other, D) or self._d != other._d + + def keys(self): + return self._d.keys() + + def values(self): + return self._d.values() + + def items(self): + return self._d.items() + + def update(self, *args, **kwargs): + self._d.update(*args, **kwargs) + + # are these part of the MutableMapping protocol? + def iterkeys(self): + return self._d.iterkeys() + + def itervalues(self): + return self._d.itervalues() + + def iteritems(self): + return self._d.iteritems() + + #def get(self, key, *args): + # return self._d.get(key, *args) + + #def pop(self, key, *args): + # return self._d.pop(key, *args) + + #def popitem(self, key): + # return self._d.popitem() + + #def clear(self): + # self._d.clear() + + #def setdefault(self, key, *args): + # return self._d.setdefault(self, key, *args) + + +kw = dict(factory=D) + +def inc(x): + return x + 1 + + +def iseven(i): + return i % 2 == 0 + + +def test_merge(): + assert merge(D({1: 1, 2: 2}), D({3: 4}), **kw) == D({1: 1, 2: 2, 3: 4}) + + +def test_merge_iterable_arg(): + assert merge([D({1: 1, 2: 2}), D({3: 4})], **kw) == D({1: 1, 2: 2, 3: 4}) + + +def test_merge_with(): + dicts = D({1: 1, 2: 2}), D({1: 10, 2: 20}) + assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22}) + assert merge_with(tuple, *dicts, **kw) == D({1: (1, 10), 2: (2, 20)}) + + dicts = D({1: 1, 2: 2, 3: 3}), D({1: 10, 2: 20}) + assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22, 3: 3}) + assert merge_with(tuple, *dicts, **kw) == D({1: (1, 10), 2: (2, 20), 3: (3,)}) + + assert not merge_with(sum) + + +def test_merge_with_iterable_arg(): + dicts = D({1: 1, 2: 2}), D({1: 10, 2: 20}) + assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22}) + assert merge_with(sum, dicts, **kw) == D({1: 11, 2: 22}) + assert merge_with(sum, iter(dicts), **kw) == D({1: 11, 2: 22}) + + +def test_valmap(): + assert valmap(inc, D({1: 1, 2: 2}), **kw) == D({1: 2, 2: 3}) + + +def test_keymap(): + assert keymap(inc, D({1: 1, 2: 2}), **kw) == D({2: 1, 3: 2}) + + +def test_itemmap(): + assert itemmap(reversed, D({1: 2, 2: 4}), **kw) == D({2: 1, 4: 2}) + + +def test_valfilter(): + assert valfilter(iseven, D({1: 2, 2: 3}), **kw) == D({1: 2}) + + +def test_keyfilter(): + assert keyfilter(iseven, D({1: 2, 2: 3}), **kw) == D({2: 3}) + + +def test_itemfilter(): + assert itemfilter(lambda item: iseven(item[0]), D({1: 2, 2: 3}), **kw) == D({2: 3}) + assert itemfilter(lambda item: iseven(item[1]), D({1: 2, 2: 3}), **kw) == D({1: 2}) + + +def test_assoc(): + assert assoc(D({}), "a", 1, **kw) == D({"a": 1}) + assert assoc(D({"a": 1}), "a", 3, **kw) == D({"a": 3}) + assert assoc(D({"a": 1}), "b", 3, **kw) == D({"a": 1, "b": 3}) + + # Verify immutability: + d = D({'x': 1}) + oldd = d + assoc(d, 'x', 2, **kw) + assert d is oldd + + +def test_dissoc(): + assert dissoc(D({"a": 1}), "a") == D({}) + assert dissoc(D({"a": 1, "b": 2}), "a") == D({"b": 2}) + assert dissoc(D({"a": 1, "b": 2}), "b") == D({"a": 1}) + + # Verify immutability: + d = D({'x': 1}) + oldd = d + d2 = dissoc(d, 'x') + assert d is oldd + assert d2 is not oldd + + +def test_update_in(): + assert update_in(D({"a": 0}), ["a"], inc, **kw) == D({"a": 1}) + assert update_in(D({"a": 0, "b": 1}), ["b"], str, **kw) == D({"a": 0, "b": "1"}) + assert (update_in(D({"t": 1, "v": D({"a": 0})}), ["v", "a"], inc, **kw) == + D({"t": 1, "v": D({"a": 1})})) + # Handle one missing key. + assert update_in(D({}), ["z"], str, None, **kw) == D({"z": "None"}) + assert update_in(D({}), ["z"], inc, 0, **kw) == D({"z": 1}) + assert update_in(D({}), ["z"], lambda x: x+"ar", default="b", **kw) == D({"z": "bar"}) + # Same semantics as Clojure for multiple missing keys, ie. recursively + # create nested empty dictionaries to the depth specified by the + # keys with the innermost value set to f(default). + assert update_in(D({}), [0, 1], inc, default=-1, **kw) == D({0: D({1: 0})}) + assert update_in(D({}), [0, 1], str, default=100, **kw) == D({0: D({1: "100"})}) + assert (update_in(D({"foo": "bar", 1: 50}), ["d", 1, 0], str, 20, **kw) == + D({"foo": "bar", 1: 50, "d": D({1: D({0: "20"})})})) + # Verify immutability: + d = D({'x': 1}) + oldd = d + update_in(d, ['x'], inc, **kw) + assert d is oldd + + +def test_factory(): + assert merge(defaultdict(int, D({1: 2})), D({2: 3})) == {1: 2, 2: 3} + assert (merge(defaultdict(int, D({1: 2})), D({2: 3}), + factory=lambda: defaultdict(int)) == + defaultdict(int, D({1: 2, 2: 3}))) + assert not (merge(defaultdict(int, D({1: 2})), D({2: 3}), + factory=lambda: defaultdict(int)) == D({1: 2, 2: 3})) + assert raises(TypeError, lambda: merge(D({1: 2}), D({2: 3}), factoryy=dict)) diff --git a/cytoolz/tests/test_dicttoolz_dict.py b/cytoolz/tests/test_dicttoolz_dict.py new file mode 100644 index 0000000..6f2225e --- /dev/null +++ b/cytoolz/tests/test_dicttoolz_dict.py @@ -0,0 +1,134 @@ +from collections import defaultdict as _defaultdict +from cytoolz.dicttoolz import (merge, merge_with, valmap, keymap, update_in, + assoc, dissoc, keyfilter, valfilter, itemmap, + itemfilter) +from cytoolz.utils import raises + + +class defaultdict(_defaultdict): + def __eq__(self, other): + return (super(defaultdict, self).__eq__(other) and + isinstance(other, _defaultdict) and + self.default_factory == other.default_factory) + + +D = dict +kw = {} + + +def inc(x): + return x + 1 + + +def iseven(i): + return i % 2 == 0 + + +def test_merge(): + assert merge(D({1: 1, 2: 2}), D({3: 4}), **kw) == D({1: 1, 2: 2, 3: 4}) + + +def test_merge_iterable_arg(): + assert merge([D({1: 1, 2: 2}), D({3: 4})], **kw) == D({1: 1, 2: 2, 3: 4}) + + +def test_merge_with(): + dicts = D({1: 1, 2: 2}), D({1: 10, 2: 20}) + assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22}) + assert merge_with(tuple, *dicts, **kw) == D({1: (1, 10), 2: (2, 20)}) + + dicts = D({1: 1, 2: 2, 3: 3}), D({1: 10, 2: 20}) + assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22, 3: 3}) + assert merge_with(tuple, *dicts, **kw) == D({1: (1, 10), 2: (2, 20), 3: (3,)}) + + assert not merge_with(sum) + + +def test_merge_with_iterable_arg(): + dicts = D({1: 1, 2: 2}), D({1: 10, 2: 20}) + assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22}) + assert merge_with(sum, dicts, **kw) == D({1: 11, 2: 22}) + assert merge_with(sum, iter(dicts), **kw) == D({1: 11, 2: 22}) + + +def test_valmap(): + assert valmap(inc, D({1: 1, 2: 2}), **kw) == D({1: 2, 2: 3}) + + +def test_keymap(): + assert keymap(inc, D({1: 1, 2: 2}), **kw) == D({2: 1, 3: 2}) + + +def test_itemmap(): + assert itemmap(reversed, D({1: 2, 2: 4}), **kw) == D({2: 1, 4: 2}) + + +def test_valfilter(): + assert valfilter(iseven, D({1: 2, 2: 3}), **kw) == D({1: 2}) + + +def test_keyfilter(): + assert keyfilter(iseven, D({1: 2, 2: 3}), **kw) == D({2: 3}) + + +def test_itemfilter(): + assert itemfilter(lambda item: iseven(item[0]), D({1: 2, 2: 3}), **kw) == D({2: 3}) + assert itemfilter(lambda item: iseven(item[1]), D({1: 2, 2: 3}), **kw) == D({1: 2}) + + +def test_assoc(): + assert assoc(D({}), "a", 1, **kw) == D({"a": 1}) + assert assoc(D({"a": 1}), "a", 3, **kw) == D({"a": 3}) + assert assoc(D({"a": 1}), "b", 3, **kw) == D({"a": 1, "b": 3}) + + # Verify immutability: + d = D({'x': 1}) + oldd = d + assoc(d, 'x', 2, **kw) + assert d is oldd + + +def test_dissoc(): + assert dissoc(D({"a": 1}), "a") == D({}) + assert dissoc(D({"a": 1, "b": 2}), "a") == D({"b": 2}) + assert dissoc(D({"a": 1, "b": 2}), "b") == D({"a": 1}) + + # Verify immutability: + d = D({'x': 1}) + oldd = d + d2 = dissoc(d, 'x') + assert d is oldd + assert d2 is not oldd + + +def test_update_in(): + assert update_in(D({"a": 0}), ["a"], inc, **kw) == D({"a": 1}) + assert update_in(D({"a": 0, "b": 1}), ["b"], str, **kw) == D({"a": 0, "b": "1"}) + assert (update_in(D({"t": 1, "v": D({"a": 0})}), ["v", "a"], inc, **kw) == + D({"t": 1, "v": D({"a": 1})})) + # Handle one missing key. + assert update_in(D({}), ["z"], str, None, **kw) == D({"z": "None"}) + assert update_in(D({}), ["z"], inc, 0, **kw) == D({"z": 1}) + assert update_in(D({}), ["z"], lambda x: x+"ar", default="b", **kw) == D({"z": "bar"}) + # Same semantics as Clojure for multiple missing keys, ie. recursively + # create nested empty dictionaries to the depth specified by the + # keys with the innermost value set to f(default). + assert update_in(D({}), [0, 1], inc, default=-1, **kw) == D({0: D({1: 0})}) + assert update_in(D({}), [0, 1], str, default=100, **kw) == D({0: D({1: "100"})}) + assert (update_in(D({"foo": "bar", 1: 50}), ["d", 1, 0], str, 20, **kw) == + D({"foo": "bar", 1: 50, "d": D({1: D({0: "20"})})})) + # Verify immutability: + d = D({'x': 1}) + oldd = d + update_in(d, ['x'], inc, **kw) + assert d is oldd + + +def test_factory(): + assert merge(defaultdict(int, D({1: 2})), D({2: 3})) == {1: 2, 2: 3} + assert (merge(defaultdict(int, D({1: 2})), D({2: 3}), + factory=lambda: defaultdict(int)) == + defaultdict(int, D({1: 2, 2: 3}))) + assert not (merge(defaultdict(int, D({1: 2})), D({2: 3}), + factory=lambda: defaultdict(int)) == D({1: 2, 2: 3})) + assert raises(TypeError, lambda: merge(D({1: 2}), D({2: 3}), factoryy=dict)) From 9ab2c016cf0235c00f4978063798314f95400448 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 20 Oct 2015 23:45:25 -0500 Subject: [PATCH 082/146] oops. these weren't forgotten after all. they were old. --- cytoolz/tests/test_dicttoolz_class.py | 237 ------------------------- cytoolz/tests/test_dicttoolz_custom.py | 198 --------------------- cytoolz/tests/test_dicttoolz_dict.py | 134 -------------- 3 files changed, 569 deletions(-) delete mode 100644 cytoolz/tests/test_dicttoolz_class.py delete mode 100644 cytoolz/tests/test_dicttoolz_custom.py delete mode 100644 cytoolz/tests/test_dicttoolz_dict.py diff --git a/cytoolz/tests/test_dicttoolz_class.py b/cytoolz/tests/test_dicttoolz_class.py deleted file mode 100644 index 7d07f60..0000000 --- a/cytoolz/tests/test_dicttoolz_class.py +++ /dev/null @@ -1,237 +0,0 @@ -from collections import defaultdict as _defaultdict -from cytoolz.dicttoolz import (merge, merge_with, valmap, keymap, update_in, - assoc, dissoc, keyfilter, valfilter, itemmap, - itemfilter) -from cytoolz.utils import raises -from cytoolz.compatibility import PY3 - - -def inc(x): - return x + 1 - - -def iseven(i): - return i % 2 == 0 - - -class TestDict(object): - """Test typical usage: dict inputs, no factory keyword. - - Class attributes: - D: callable that inputs a dict and creates or returns a MutableMapping - kw: kwargs dict to specify "factory" keyword (if applicable) - """ - D = dict - kw = {} - - def test_merge(self): - D, kw = self.D, self.kw - assert merge(D({1: 1, 2: 2}), D({3: 4}), **kw) == D({1: 1, 2: 2, 3: 4}) - - def test_merge_iterable_arg(self): - D, kw = self.D, self.kw - assert merge([D({1: 1, 2: 2}), D({3: 4})], **kw) == D({1: 1, 2: 2, 3: 4}) - - def test_merge_with(self): - D, kw = self.D, self.kw - dicts = D({1: 1, 2: 2}), D({1: 10, 2: 20}) - assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22}) - assert merge_with(tuple, *dicts, **kw) == D({1: (1, 10), 2: (2, 20)}) - - dicts = D({1: 1, 2: 2, 3: 3}), D({1: 10, 2: 20}) - assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22, 3: 3}) - assert merge_with(tuple, *dicts, **kw) == D({1: (1, 10), 2: (2, 20), 3: (3,)}) - - assert not merge_with(sum) - - def test_merge_with_iterable_arg(self): - D, kw = self.D, self.kw - dicts = D({1: 1, 2: 2}), D({1: 10, 2: 20}) - assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22}) - assert merge_with(sum, dicts, **kw) == D({1: 11, 2: 22}) - assert merge_with(sum, iter(dicts), **kw) == D({1: 11, 2: 22}) - - def test_valmap(self): - D, kw = self.D, self.kw - assert valmap(inc, D({1: 1, 2: 2}), **kw) == D({1: 2, 2: 3}) - - def test_keymap(self): - D, kw = self.D, self.kw - assert keymap(inc, D({1: 1, 2: 2}), **kw) == D({2: 1, 3: 2}) - - def test_itemmap(self): - D, kw = self.D, self.kw - assert itemmap(reversed, D({1: 2, 2: 4}), **kw) == D({2: 1, 4: 2}) - - def test_valfilter(self): - D, kw = self.D, self.kw - assert valfilter(iseven, D({1: 2, 2: 3}), **kw) == D({1: 2}) - - def test_keyfilter(self): - D, kw = self.D, self.kw - assert keyfilter(iseven, D({1: 2, 2: 3}), **kw) == D({2: 3}) - - def test_itemfilter(self): - D, kw = self.D, self.kw - assert itemfilter(lambda item: iseven(item[0]), D({1: 2, 2: 3}), **kw) == D({2: 3}) - assert itemfilter(lambda item: iseven(item[1]), D({1: 2, 2: 3}), **kw) == D({1: 2}) - - def test_assoc(self): - D, kw = self.D, self.kw - assert assoc(D({}), "a", 1, **kw) == D({"a": 1}) - assert assoc(D({"a": 1}), "a", 3, **kw) == D({"a": 3}) - assert assoc(D({"a": 1}), "b", 3, **kw) == D({"a": 1, "b": 3}) - - # Verify immutability: - d = D({'x': 1}) - oldd = d - assoc(d, 'x', 2, **kw) - assert d is oldd - - def test_dissoc(self): - D, kw = self.D, self.kw - assert dissoc(D({"a": 1}), "a") == D({}) - assert dissoc(D({"a": 1, "b": 2}), "a") == D({"b": 2}) - assert dissoc(D({"a": 1, "b": 2}), "b") == D({"a": 1}) - - # Verify immutability: - d = D({'x': 1}) - oldd = d - d2 = dissoc(d, 'x') - assert d is oldd - assert d2 is not oldd - - def test_update_in(self): - D, kw = self.D, self.kw - assert update_in(D({"a": 0}), ["a"], inc, **kw) == D({"a": 1}) - assert update_in(D({"a": 0, "b": 1}), ["b"], str, **kw) == D({"a": 0, "b": "1"}) - assert (update_in(D({"t": 1, "v": D({"a": 0})}), ["v", "a"], inc, **kw) == - D({"t": 1, "v": D({"a": 1})})) - # Handle one missing key. - assert update_in(D({}), ["z"], str, None, **kw) == D({"z": "None"}) - assert update_in(D({}), ["z"], inc, 0, **kw) == D({"z": 1}) - assert update_in(D({}), ["z"], lambda x: x+"ar", default="b", **kw) == D({"z": "bar"}) - # Same semantics as Clojure for multiple missing keys, ie. recursively - # create nested empty dictionaries to the depth specified by the - # keys with the innermost value set to f(default). - assert update_in(D({}), [0, 1], inc, default=-1, **kw) == D({0: D({1: 0})}) - assert update_in(D({}), [0, 1], str, default=100, **kw) == D({0: D({1: "100"})}) - assert (update_in(D({"foo": "bar", 1: 50}), ["d", 1, 0], str, 20, **kw) == - D({"foo": "bar", 1: 50, "d": D({1: D({0: "20"})})})) - # Verify immutability: - d = D({'x': 1}) - oldd = d - update_in(d, ['x'], inc, **kw) - assert d is oldd - - def test_factory(self): - D, kw = self.D, self.kw - assert merge(defaultdict(int, D({1: 2})), D({2: 3})) == {1: 2, 2: 3} - assert (merge(defaultdict(int, D({1: 2})), D({2: 3}), - factory=lambda: defaultdict(int)) == - defaultdict(int, D({1: 2, 2: 3}))) - assert not (merge(defaultdict(int, D({1: 2})), D({2: 3}), - factory=lambda: defaultdict(int)) == {1: 2, 2: 3}) - assert raises(TypeError, lambda: merge(D({1: 2}), D({2: 3}), factoryy=dict)) - - -class defaultdict(_defaultdict): - def __eq__(self, other): - return (super(defaultdict, self).__eq__(other) and - isinstance(other, _defaultdict) and - self.default_factory == other.default_factory) - - -class TestDefaultDict(TestDict): - """Test defaultdict as input and factory - - Class attributes: - D: callable that inputs a dict and creates or returns a MutableMapping - kw: kwargs dict to specify "factory" keyword (if applicable) - """ - @staticmethod - def D(dict_): - return defaultdict(int, dict_) - - kw = {'factory': lambda: defaultdict(int)} - - -class CustomMapping(object): - """Define methods of the MutableMapping protocol required by dicttoolz""" - def __init__(self, *args, **kwargs): - self._d = dict(*args, **kwargs) - - def __getitem__(self, key): - return self._d[key] - - def __setitem__(self, key, val): - self._d[key] = val - - def __delitem__(self, key): - del self._d[key] - - def __iter__(self): - return iter(self._d) - - def __len__(self): - return len(self._d) - - def __contains__(self, key): - return key in self._d - - def __eq__(self, other): - return isinstance(other, CustomMapping) and self._d == other._d - - def __ne__(self, other): - return not isinstance(other, CustomMapping) or self._d != other._d - - def keys(self): - return self._d.keys() - - def values(self): - return self._d.values() - - def items(self): - return self._d.items() - - def update(self, *args, **kwargs): - self._d.update(*args, **kwargs) - - # Should we require these to be defined for Python 2? - if not PY3: - def iterkeys(self): - return self._d.iterkeys() - - def itervalues(self): - return self._d.itervalues() - - def iteritems(self): - return self._d.iteritems() - - # Unused methods that are part of the MutableMapping protocol - #def get(self, key, *args): - # return self._d.get(key, *args) - - #def pop(self, key, *args): - # return self._d.pop(key, *args) - - #def popitem(self, key): - # return self._d.popitem() - - #def clear(self): - # self._d.clear() - - #def setdefault(self, key, *args): - # return self._d.setdefault(self, key, *args) - - -class TestCustomMapping(TestDict): - """Test CustomMapping as input and factory - - Class attributes: - D: callable that inputs a dict and creates or returns a MutableMapping - kw: kwargs dict to specify "factory" keyword (if applicable) - """ - D = CustomMapping - kw = {'factory': lambda: CustomMapping()} - diff --git a/cytoolz/tests/test_dicttoolz_custom.py b/cytoolz/tests/test_dicttoolz_custom.py deleted file mode 100644 index 0cfe123..0000000 --- a/cytoolz/tests/test_dicttoolz_custom.py +++ /dev/null @@ -1,198 +0,0 @@ -from collections import defaultdict as _defaultdict -from cytoolz.dicttoolz import (merge, merge_with, valmap, keymap, update_in, - assoc, dissoc, keyfilter, valfilter, itemmap, - itemfilter) -from cytoolz.utils import raises - - -class defaultdict(_defaultdict): - def __eq__(self, other): - return (super(defaultdict, self).__eq__(other) and - isinstance(other, _defaultdict) and - self.default_factory == other.default_factory) - - -class D(object): - def __init__(self, *args, **kwargs): - self._d = dict(*args, **kwargs) - - def __getitem__(self, key): - return self._d[key] - - def __setitem__(self, key, val): - self._d[key] = val - - def __delitem__(self, key): - del self._d[key] - - def __iter__(self): - return iter(self._d) - - def __len__(self): - return len(self._d) - - def __contains__(self, key): - return key in self._d - - def __eq__(self, other): - return isinstance(other, D) and self._d == other._d - - def __ne__(self, other): - return not isinstance(other, D) or self._d != other._d - - def keys(self): - return self._d.keys() - - def values(self): - return self._d.values() - - def items(self): - return self._d.items() - - def update(self, *args, **kwargs): - self._d.update(*args, **kwargs) - - # are these part of the MutableMapping protocol? - def iterkeys(self): - return self._d.iterkeys() - - def itervalues(self): - return self._d.itervalues() - - def iteritems(self): - return self._d.iteritems() - - #def get(self, key, *args): - # return self._d.get(key, *args) - - #def pop(self, key, *args): - # return self._d.pop(key, *args) - - #def popitem(self, key): - # return self._d.popitem() - - #def clear(self): - # self._d.clear() - - #def setdefault(self, key, *args): - # return self._d.setdefault(self, key, *args) - - -kw = dict(factory=D) - -def inc(x): - return x + 1 - - -def iseven(i): - return i % 2 == 0 - - -def test_merge(): - assert merge(D({1: 1, 2: 2}), D({3: 4}), **kw) == D({1: 1, 2: 2, 3: 4}) - - -def test_merge_iterable_arg(): - assert merge([D({1: 1, 2: 2}), D({3: 4})], **kw) == D({1: 1, 2: 2, 3: 4}) - - -def test_merge_with(): - dicts = D({1: 1, 2: 2}), D({1: 10, 2: 20}) - assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22}) - assert merge_with(tuple, *dicts, **kw) == D({1: (1, 10), 2: (2, 20)}) - - dicts = D({1: 1, 2: 2, 3: 3}), D({1: 10, 2: 20}) - assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22, 3: 3}) - assert merge_with(tuple, *dicts, **kw) == D({1: (1, 10), 2: (2, 20), 3: (3,)}) - - assert not merge_with(sum) - - -def test_merge_with_iterable_arg(): - dicts = D({1: 1, 2: 2}), D({1: 10, 2: 20}) - assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22}) - assert merge_with(sum, dicts, **kw) == D({1: 11, 2: 22}) - assert merge_with(sum, iter(dicts), **kw) == D({1: 11, 2: 22}) - - -def test_valmap(): - assert valmap(inc, D({1: 1, 2: 2}), **kw) == D({1: 2, 2: 3}) - - -def test_keymap(): - assert keymap(inc, D({1: 1, 2: 2}), **kw) == D({2: 1, 3: 2}) - - -def test_itemmap(): - assert itemmap(reversed, D({1: 2, 2: 4}), **kw) == D({2: 1, 4: 2}) - - -def test_valfilter(): - assert valfilter(iseven, D({1: 2, 2: 3}), **kw) == D({1: 2}) - - -def test_keyfilter(): - assert keyfilter(iseven, D({1: 2, 2: 3}), **kw) == D({2: 3}) - - -def test_itemfilter(): - assert itemfilter(lambda item: iseven(item[0]), D({1: 2, 2: 3}), **kw) == D({2: 3}) - assert itemfilter(lambda item: iseven(item[1]), D({1: 2, 2: 3}), **kw) == D({1: 2}) - - -def test_assoc(): - assert assoc(D({}), "a", 1, **kw) == D({"a": 1}) - assert assoc(D({"a": 1}), "a", 3, **kw) == D({"a": 3}) - assert assoc(D({"a": 1}), "b", 3, **kw) == D({"a": 1, "b": 3}) - - # Verify immutability: - d = D({'x': 1}) - oldd = d - assoc(d, 'x', 2, **kw) - assert d is oldd - - -def test_dissoc(): - assert dissoc(D({"a": 1}), "a") == D({}) - assert dissoc(D({"a": 1, "b": 2}), "a") == D({"b": 2}) - assert dissoc(D({"a": 1, "b": 2}), "b") == D({"a": 1}) - - # Verify immutability: - d = D({'x': 1}) - oldd = d - d2 = dissoc(d, 'x') - assert d is oldd - assert d2 is not oldd - - -def test_update_in(): - assert update_in(D({"a": 0}), ["a"], inc, **kw) == D({"a": 1}) - assert update_in(D({"a": 0, "b": 1}), ["b"], str, **kw) == D({"a": 0, "b": "1"}) - assert (update_in(D({"t": 1, "v": D({"a": 0})}), ["v", "a"], inc, **kw) == - D({"t": 1, "v": D({"a": 1})})) - # Handle one missing key. - assert update_in(D({}), ["z"], str, None, **kw) == D({"z": "None"}) - assert update_in(D({}), ["z"], inc, 0, **kw) == D({"z": 1}) - assert update_in(D({}), ["z"], lambda x: x+"ar", default="b", **kw) == D({"z": "bar"}) - # Same semantics as Clojure for multiple missing keys, ie. recursively - # create nested empty dictionaries to the depth specified by the - # keys with the innermost value set to f(default). - assert update_in(D({}), [0, 1], inc, default=-1, **kw) == D({0: D({1: 0})}) - assert update_in(D({}), [0, 1], str, default=100, **kw) == D({0: D({1: "100"})}) - assert (update_in(D({"foo": "bar", 1: 50}), ["d", 1, 0], str, 20, **kw) == - D({"foo": "bar", 1: 50, "d": D({1: D({0: "20"})})})) - # Verify immutability: - d = D({'x': 1}) - oldd = d - update_in(d, ['x'], inc, **kw) - assert d is oldd - - -def test_factory(): - assert merge(defaultdict(int, D({1: 2})), D({2: 3})) == {1: 2, 2: 3} - assert (merge(defaultdict(int, D({1: 2})), D({2: 3}), - factory=lambda: defaultdict(int)) == - defaultdict(int, D({1: 2, 2: 3}))) - assert not (merge(defaultdict(int, D({1: 2})), D({2: 3}), - factory=lambda: defaultdict(int)) == D({1: 2, 2: 3})) - assert raises(TypeError, lambda: merge(D({1: 2}), D({2: 3}), factoryy=dict)) diff --git a/cytoolz/tests/test_dicttoolz_dict.py b/cytoolz/tests/test_dicttoolz_dict.py deleted file mode 100644 index 6f2225e..0000000 --- a/cytoolz/tests/test_dicttoolz_dict.py +++ /dev/null @@ -1,134 +0,0 @@ -from collections import defaultdict as _defaultdict -from cytoolz.dicttoolz import (merge, merge_with, valmap, keymap, update_in, - assoc, dissoc, keyfilter, valfilter, itemmap, - itemfilter) -from cytoolz.utils import raises - - -class defaultdict(_defaultdict): - def __eq__(self, other): - return (super(defaultdict, self).__eq__(other) and - isinstance(other, _defaultdict) and - self.default_factory == other.default_factory) - - -D = dict -kw = {} - - -def inc(x): - return x + 1 - - -def iseven(i): - return i % 2 == 0 - - -def test_merge(): - assert merge(D({1: 1, 2: 2}), D({3: 4}), **kw) == D({1: 1, 2: 2, 3: 4}) - - -def test_merge_iterable_arg(): - assert merge([D({1: 1, 2: 2}), D({3: 4})], **kw) == D({1: 1, 2: 2, 3: 4}) - - -def test_merge_with(): - dicts = D({1: 1, 2: 2}), D({1: 10, 2: 20}) - assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22}) - assert merge_with(tuple, *dicts, **kw) == D({1: (1, 10), 2: (2, 20)}) - - dicts = D({1: 1, 2: 2, 3: 3}), D({1: 10, 2: 20}) - assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22, 3: 3}) - assert merge_with(tuple, *dicts, **kw) == D({1: (1, 10), 2: (2, 20), 3: (3,)}) - - assert not merge_with(sum) - - -def test_merge_with_iterable_arg(): - dicts = D({1: 1, 2: 2}), D({1: 10, 2: 20}) - assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22}) - assert merge_with(sum, dicts, **kw) == D({1: 11, 2: 22}) - assert merge_with(sum, iter(dicts), **kw) == D({1: 11, 2: 22}) - - -def test_valmap(): - assert valmap(inc, D({1: 1, 2: 2}), **kw) == D({1: 2, 2: 3}) - - -def test_keymap(): - assert keymap(inc, D({1: 1, 2: 2}), **kw) == D({2: 1, 3: 2}) - - -def test_itemmap(): - assert itemmap(reversed, D({1: 2, 2: 4}), **kw) == D({2: 1, 4: 2}) - - -def test_valfilter(): - assert valfilter(iseven, D({1: 2, 2: 3}), **kw) == D({1: 2}) - - -def test_keyfilter(): - assert keyfilter(iseven, D({1: 2, 2: 3}), **kw) == D({2: 3}) - - -def test_itemfilter(): - assert itemfilter(lambda item: iseven(item[0]), D({1: 2, 2: 3}), **kw) == D({2: 3}) - assert itemfilter(lambda item: iseven(item[1]), D({1: 2, 2: 3}), **kw) == D({1: 2}) - - -def test_assoc(): - assert assoc(D({}), "a", 1, **kw) == D({"a": 1}) - assert assoc(D({"a": 1}), "a", 3, **kw) == D({"a": 3}) - assert assoc(D({"a": 1}), "b", 3, **kw) == D({"a": 1, "b": 3}) - - # Verify immutability: - d = D({'x': 1}) - oldd = d - assoc(d, 'x', 2, **kw) - assert d is oldd - - -def test_dissoc(): - assert dissoc(D({"a": 1}), "a") == D({}) - assert dissoc(D({"a": 1, "b": 2}), "a") == D({"b": 2}) - assert dissoc(D({"a": 1, "b": 2}), "b") == D({"a": 1}) - - # Verify immutability: - d = D({'x': 1}) - oldd = d - d2 = dissoc(d, 'x') - assert d is oldd - assert d2 is not oldd - - -def test_update_in(): - assert update_in(D({"a": 0}), ["a"], inc, **kw) == D({"a": 1}) - assert update_in(D({"a": 0, "b": 1}), ["b"], str, **kw) == D({"a": 0, "b": "1"}) - assert (update_in(D({"t": 1, "v": D({"a": 0})}), ["v", "a"], inc, **kw) == - D({"t": 1, "v": D({"a": 1})})) - # Handle one missing key. - assert update_in(D({}), ["z"], str, None, **kw) == D({"z": "None"}) - assert update_in(D({}), ["z"], inc, 0, **kw) == D({"z": 1}) - assert update_in(D({}), ["z"], lambda x: x+"ar", default="b", **kw) == D({"z": "bar"}) - # Same semantics as Clojure for multiple missing keys, ie. recursively - # create nested empty dictionaries to the depth specified by the - # keys with the innermost value set to f(default). - assert update_in(D({}), [0, 1], inc, default=-1, **kw) == D({0: D({1: 0})}) - assert update_in(D({}), [0, 1], str, default=100, **kw) == D({0: D({1: "100"})}) - assert (update_in(D({"foo": "bar", 1: 50}), ["d", 1, 0], str, 20, **kw) == - D({"foo": "bar", 1: 50, "d": D({1: D({0: "20"})})})) - # Verify immutability: - d = D({'x': 1}) - oldd = d - update_in(d, ['x'], inc, **kw) - assert d is oldd - - -def test_factory(): - assert merge(defaultdict(int, D({1: 2})), D({2: 3})) == {1: 2, 2: 3} - assert (merge(defaultdict(int, D({1: 2})), D({2: 3}), - factory=lambda: defaultdict(int)) == - defaultdict(int, D({1: 2, 2: 3}))) - assert not (merge(defaultdict(int, D({1: 2})), D({2: 3}), - factory=lambda: defaultdict(int)) == D({1: 2, 2: 3})) - assert raises(TypeError, lambda: merge(D({1: 2}), D({2: 3}), factoryy=dict)) From 0edcfe045cdd65cc4be8719a29f29a1f4a7c7975 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 20 Oct 2015 23:47:22 -0500 Subject: [PATCH 083/146] update conda version and add Python 3.5 to tests --- .travis.yml | 1 + conda.recipe/meta.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1686736..87ca558 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - "3.2" - "3.3" - "3.4" + - "3.5" before_install: - pip install git+https://github.com/pytoolz/toolz.git diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index b535090..a3a3e11 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: cytoolz - version: "0.7.3" + version: "0.7.4" build: number: {{environ.get('BINSTAR_BUILD', 1)}} From ec820f707aa9dcb9456376930308f4f9f3bb20d4 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 20 Oct 2015 23:54:36 -0500 Subject: [PATCH 084/146] punt on python 3.5 for now --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 87ca558..1686736 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ python: - "3.2" - "3.3" - "3.4" - - "3.5" before_install: - pip install git+https://github.com/pytoolz/toolz.git From cc91ec7e92b8329119d7088a49f49c57a4b70f88 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 21 Oct 2015 00:16:48 -0500 Subject: [PATCH 085/146] Update setup.py, because cytoolz.curried is a package now. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3663b7f..db8ddf6 100644 --- a/setup.py +++ b/setup.py @@ -78,8 +78,8 @@ maintainer='Erik Welch', maintainer_email='erik.n.welch@gmail.com', license = 'BSD', - packages=['cytoolz'], - package_data={'cytoolz': ['*.pyx', '*.pxd', 'tests/*.py']}, + packages=['cytoolz', 'cytoolz.curried'], + package_data={'cytoolz': ['*.pyx', '*.pxd', 'curried/*.pyx', 'tests/*.py']}, # include_package_data = True, keywords=('functional utility itertools functools iterator generator ' 'curry memoize lazy streaming bigdata cython toolz cytoolz'), From 8c92e0d63950cab30b185205218e359b7ca1ffaf Mon Sep 17 00:00:00 2001 From: shearerp Date: Sun, 3 Jan 2016 07:55:19 -0500 Subject: [PATCH 086/146] typo --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index ff7521e..42df745 100644 --- a/README.rst +++ b/README.rst @@ -20,7 +20,7 @@ usage is achieved by using the iterator protocol and returning iterators whenever possible. ``cytoolz`` implements the same API as ``toolz``. The main differences are -that ``cytoolz`` is faster (typically 2-5x faster with a few spectactular +that ``cytoolz`` is faster (typically 2-5x faster with a few spectacular exceptions) and ``cytoolz`` offers a C API that is accessible to other projects developed in Cython. Since ``toolz`` is able to process very large (potentially infinite) data sets, the performance increase gained by From b769472e7ca76649e0628c2026a35472f932f42a Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 4 Jan 2016 12:04:02 -0500 Subject: [PATCH 087/146] bump to dev version --- cytoolz/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cytoolz/_version.py b/cytoolz/_version.py index 7d0d165..aba9e08 100644 --- a/cytoolz/_version.py +++ b/cytoolz/_version.py @@ -1,2 +1,2 @@ -__version__ = '0.7.4' +__version__ = '0.7.5dev' __toolz_version__ = '0.7.4' From 41df823593dd26682fea56ee78b140f51e6b456f Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 26 Jan 2016 17:56:32 -0600 Subject: [PATCH 088/146] Add Python 3.5 support. Fix issue where exceptions need to be cleared before re-raising them. Fixes #73 --- .travis.yml | 1 + cytoolz/dicttoolz.pyx | 2 +- cytoolz/itertoolz.pyx | 22 ++++++++++++---------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1686736..87ca558 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - "3.2" - "3.3" - "3.4" + - "3.5" before_install: - pip install git+https://github.com/pytoolz/toolz.git diff --git a/cytoolz/dicttoolz.pyx b/cytoolz/dicttoolz.pyx index 38ad58d..8814560 100644 --- a/cytoolz/dicttoolz.pyx +++ b/cytoolz/dicttoolz.pyx @@ -506,9 +506,9 @@ cpdef object get_in(object keys, object coll, object default=None, object no_def obj = PtrObject_GetItem(coll, item) if obj is NULL: item = PyErr_Occurred() + PyErr_Clear() if no_default or not PyErr_GivenExceptionMatches(item, _get_in_exceptions): raise item - PyErr_Clear() return default Py_XDECREF(obj) coll = obj diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 65d6e51..4fcb5ed 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -1,8 +1,7 @@ #cython: embedsignature=True from cpython.dict cimport PyDict_GetItem, PyDict_SetItem -from cpython.exc cimport (PyErr_Clear, PyErr_ExceptionMatches, - PyErr_GivenExceptionMatches, PyErr_Occurred) -from cpython.list cimport (PyList_Append, PyList_GET_ITEM, PyList_GET_SIZE) +from cpython.exc cimport PyErr_Clear, PyErr_GivenExceptionMatches, PyErr_Occurred +from cpython.list cimport PyList_Append, PyList_GET_ITEM, PyList_GET_SIZE from cpython.object cimport PyObject_RichCompareBool, Py_NE from cpython.ref cimport PyObject, Py_INCREF, Py_XDECREF from cpython.sequence cimport PySequence_Check @@ -663,9 +662,10 @@ cpdef object get(object ind, object seq, object default=no_default): for i, val in enumerate(ind): obj = PtrObject_GetItem(seq, val) if obj is NULL: - if not PyErr_ExceptionMatches(_get_list_exc): - raise PyErr_Occurred() + val = PyErr_Occurred() PyErr_Clear() + if not PyErr_GivenExceptionMatches(val, _get_list_exc): + raise val Py_INCREF(default) PyTuple_SET_ITEM(result, i, default) else: @@ -676,10 +676,10 @@ cpdef object get(object ind, object seq, object default=no_default): obj = PtrObject_GetItem(seq, ind) if obj is NULL: val = PyErr_Occurred() + PyErr_Clear() if default is no_default: raise val if PyErr_GivenExceptionMatches(val, _get_exceptions): - PyErr_Clear() return default raise val Py_XDECREF(obj) @@ -1065,9 +1065,10 @@ cdef class _pluck_index_default: val = next(self.iterseqs) obj = PtrObject_GetItem(val, self.ind) if obj is NULL: - if not PyErr_ExceptionMatches(_get_exceptions): - raise PyErr_Occurred() + val = PyErr_Occurred() PyErr_Clear() + if not PyErr_GivenExceptionMatches(val, _get_exceptions): + raise val return self.default Py_XDECREF(obj) return obj @@ -1114,9 +1115,10 @@ cdef class _pluck_list_default: for i, val in enumerate(self.ind): obj = PtrObject_GetItem(seq, val) if obj is NULL: - if not PyErr_ExceptionMatches(_get_list_exc): - raise PyErr_Occurred() + val = PyErr_Occurred() PyErr_Clear() + if not PyErr_GivenExceptionMatches(val, _get_list_exc): + raise val Py_INCREF(self.default) PyTuple_SET_ITEM(result, i, self.default) else: From ab07d105eed08db76cdf46b238e6963fd64119aa Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 26 Jan 2016 18:19:50 -0600 Subject: [PATCH 089/146] oops, missed a couple places where errors should be cleared. --- cytoolz/functoolz.pyx | 6 +++--- cytoolz/itertoolz.pyx | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 4e252b2..aadeefb 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -5,7 +5,7 @@ from functools import partial from cytoolz.compatibility import filter as ifilter, map as imap, reduce from cpython.dict cimport PyDict_Merge, PyDict_New -from cpython.exc cimport PyErr_Clear, PyErr_ExceptionMatches, PyErr_Occurred +from cpython.exc cimport PyErr_Clear, PyErr_Occurred, PyErr_GivenExceptionMatches from cpython.object cimport (PyCallable_Check, PyObject_Call, PyObject_CallObject, PyObject_RichCompare, Py_EQ, Py_NE) from cpython.ref cimport PyObject, Py_DECREF @@ -266,8 +266,8 @@ cdef class curry: return val val = PyErr_Occurred() - if PyErr_ExceptionMatches(TypeError): - PyErr_Clear() + PyErr_Clear() + if PyErr_GivenExceptionMatches(val, TypeError): required_args = _num_required_args(self.func) # If there was a genuine TypeError if required_args == -1 or len(args) < required_args: diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 4fcb5ed..575c72b 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -355,9 +355,9 @@ cdef class interleave: obj = PyErr_Occurred() if obj is not NULL: val = obj + PyErr_Clear() if not PyErr_GivenExceptionMatches(val, self.pass_exceptions): raise val - PyErr_Clear() if self.i == self.n: self.n = PyList_GET_SIZE(self.newiters) From 6320049f913ebbcb2366a20178289b05d3e1e4c5 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 26 Jan 2016 22:59:49 -0600 Subject: [PATCH 090/146] bump version to 0.7.6dev. 0.7.5 was just released. --- cytoolz/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cytoolz/_version.py b/cytoolz/_version.py index aba9e08..7e05c3a 100644 --- a/cytoolz/_version.py +++ b/cytoolz/_version.py @@ -1,2 +1,2 @@ -__version__ = '0.7.5dev' +__version__ = '0.7.6dev' __toolz_version__ = '0.7.4' From 707d65065db88b5f99c19049a2c722f8c98bdd8d Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 17 Apr 2016 18:38:36 -0500 Subject: [PATCH 091/146] Up-to-date with toolz: assoc_in, excepts, random_sample, is_partial_args, is_valid_args Also updated behavior of curry and dissoc. --- .travis.yml | 1 - cytoolz/__init__.pxd | 6 +- cytoolz/_signatures.py | 774 ++++++++++++++++++++++++++++ cytoolz/compatibility.py | 1 + cytoolz/curried/__init__.py | 3 + cytoolz/curried/exceptions.pyx | 1 - cytoolz/dicttoolz.pxd | 3 + cytoolz/dicttoolz.pyx | 49 +- cytoolz/functoolz.pxd | 17 + cytoolz/functoolz.pyx | 356 +++++++++++-- cytoolz/itertoolz.pxd | 6 + cytoolz/itertoolz.pyx | 56 +- cytoolz/recipes.pyx | 1 - cytoolz/tests/test_dicttoolz.py | 18 +- cytoolz/tests/test_embedded_sigs.py | 2 + cytoolz/tests/test_functoolz.py | 158 +++++- cytoolz/tests/test_inspect_args.py | 267 ++++++++++ cytoolz/tests/test_itertoolz.py | 31 +- cytoolz/tests/test_none_safe.py | 12 + cytoolz/tests/test_signatures.py | 81 +++ cytoolz/utils.pyx | 1 - setup.py | 5 +- 22 files changed, 1771 insertions(+), 78 deletions(-) create mode 100644 cytoolz/_signatures.py create mode 100644 cytoolz/tests/test_inspect_args.py create mode 100644 cytoolz/tests/test_signatures.py diff --git a/.travis.yml b/.travis.yml index 87ca558..37137fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: python python: - "2.6" - "2.7" - - "3.2" - "3.3" - "3.4" - "3.5" diff --git a/cytoolz/__init__.pxd b/cytoolz/__init__.pxd index 4146561..87b3a7b 100644 --- a/cytoolz/__init__.pxd +++ b/cytoolz/__init__.pxd @@ -3,17 +3,17 @@ from cytoolz.itertoolz cimport ( frequencies, interleave, interpose, isdistinct, isiterable, iterate, last, mapcat, nth, partition, partition_all, pluck, reduceby, remove, rest, second, sliding_window, take, tail, take_nth, unique, join, - c_diff, topk, peek) + c_diff, topk, peek, random_sample) from cytoolz.functoolz cimport ( c_compose, c_juxt, c_memoize, c_pipe, c_thread_first, c_thread_last, - complement, curry, do, identity, memoize) + complement, curry, do, identity, memoize, excepts) from cytoolz.dicttoolz cimport ( assoc, c_merge, c_merge_with, c_dissoc, get_in, keyfilter, keymap, - itemfilter, itemmap, update_in, valfilter, valmap) + itemfilter, itemmap, update_in, valfilter, valmap, assoc_in) from cytoolz.recipes cimport countby, partitionby diff --git a/cytoolz/_signatures.py b/cytoolz/_signatures.py new file mode 100644 index 0000000..340fdc6 --- /dev/null +++ b/cytoolz/_signatures.py @@ -0,0 +1,774 @@ +"""Internal module for better introspection of builtins. + +The main functions are ``is_builtin_valid_args``, ``is_builtin_partial_args``, +and ``has_unknown_args``. Other functions in this module support these three. + +Notably, we create a ``signatures`` registry to enable introspection of +builtin functions in any Python version. This includes builtins that +have more than one valid signature. Currently, the registry includes +builtins from ``builtins``, ``functools``, ``itertools``, and ``operator`` +modules. More can be added as requested. We don't guarantee full coverage. + +Everything in this module should be regarded as implementation details. +Users should try to not use this module directly. + +""" +import functools +import inspect +import itertools +import operator +import sys + +from cytoolz.compatibility import PY3 + +if PY3: # pragma: py2 no cover + import builtins +else: # pragma: py3 no cover + import __builtin__ as builtins + +# We mock builtin callables using lists of tuples with lambda functions. +# +# The tuple spec is (num_position_args, lambda_func, keyword_only_args). +# +# num_position_args: +# - The number of positional-only arguments. If not specified, +# all positional arguments are considered positional-only. +# +# lambda_func: +# - lambda function that matches a signature of a builtin, but does +# not include keyword-only arguments. +# +# keyword_only_args: (optional) +# - Tuple of keyword-only argumemts. + +module_info = {} + +module_info[builtins] = dict( + abs=[ + lambda x: None], + all=[ + lambda iterable: None], + any=[ + lambda iterable: None], + apply=[ + lambda object: None, + lambda object, args: None, + lambda object, args, kwargs: None], + ascii=[ + lambda obj: None], + bin=[ + lambda number: None], + bool=[ + lambda x=False: None], + buffer=[ + lambda object: None, + lambda object, offset: None, + lambda object, offset, size: None], + bytearray=[ + lambda: None, + lambda int: None, + lambda string, encoding='utf8', errors='strict': None], + callable=[ + lambda obj: None], + chr=[ + lambda i: None], + classmethod=[ + lambda function: None], + cmp=[ + lambda x, y: None], + coerce=[ + lambda x, y: None], + complex=[ + lambda real=0, imag=0: None], + delattr=[ + lambda obj, name: None], + dict=[ + lambda **kwargs: None, + lambda mapping, **kwargs: None], + dir=[ + lambda: None, + lambda object: None], + divmod=[ + lambda x, y: None], + enumerate=[ + (0, lambda iterable, start=0: None)], + eval=[ + lambda source: None, + lambda source, globals: None, + lambda source, globals, locals: None], + execfile=[ + lambda filename: None, + lambda filename, globals: None, + lambda filename, globals, locals: None], + file=[ + (0, lambda name, mode='r', buffering=-1: None)], + filter=[ + lambda function, iterable: None], + float=[ + lambda x=0.0: None], + format=[ + lambda value: None, + lambda value, format_spec: None], + frozenset=[ + lambda: None, + lambda iterable: None], + getattr=[ + lambda object, name: None, + lambda object, name, default: None], + globals=[ + lambda: None], + hasattr=[ + lambda obj, name: None], + hash=[ + lambda obj: None], + hex=[ + lambda number: None], + id=[ + lambda obj: None], + input=[ + lambda: None, + lambda prompt: None], + int=[ + lambda x=0: None, + (0, lambda x, base=10: None)], + intern=[ + lambda string: None], + isinstance=[ + lambda obj, class_or_tuple: None], + issubclass=[ + lambda cls, class_or_tuple: None], + iter=[ + lambda iterable: None, + lambda callable, sentinel: None], + len=[ + lambda obj: None], + list=[ + lambda: None, + lambda iterable: None], + locals=[ + lambda: None], + long=[ + lambda x=0: None, + (0, lambda x, base=10: None)], + map=[ + lambda func, sequence, *iterables: None], + memoryview=[ + (0, lambda object: None)], + next=[ + lambda iterator: None, + lambda iterator, default: None], + object=[ + lambda: None], + oct=[ + lambda number: None], + ord=[ + lambda c: None], + pow=[ + lambda x, y: None, + lambda x, y, z: None], + property=[ + lambda fget=None, fset=None, fdel=None, doc=None: None], + range=[ + lambda stop: None, + lambda start, stop: None, + lambda start, stop, step: None], + raw_input=[ + lambda: None, + lambda prompt: None], + reduce=[ + lambda function, sequence: None, + lambda function, sequence, initial: None], + reload=[ + lambda module: None], + repr=[ + lambda obj: None], + reversed=[ + lambda sequence: None], + round=[ + (0, lambda number, ndigits=0: None)], + set=[ + lambda: None, + lambda iterable: None], + setattr=[ + lambda obj, name, value: None], + slice=[ + lambda stop: None, + lambda start, stop: None, + lambda start, stop, step: None], + staticmethod=[ + lambda function: None], + sum=[ + lambda iterable: None, + lambda iterable, start: None], + super=[ + lambda type: None, + lambda type, obj: None], + tuple=[ + lambda: None, + lambda iterable: None], + type=[ + lambda object: None, + lambda name, bases, dict: None], + unichr=[ + lambda i: None], + unicode=[ + lambda object: None, + lambda string='', encoding='utf8', errors='strict': None], + vars=[ + lambda: None, + lambda object: None], + xrange=[ + lambda stop: None, + lambda start, stop: None, + lambda start, stop, step: None], + zip=[ + lambda *iterables: None], +) +module_info[builtins]['exec'] = [ + lambda source: None, + lambda source, globals: None, + lambda source, globals, locals: None] + +if PY3: # pragma: py2 no cover + module_info[builtins].update( + bytes=[ + lambda: None, + lambda int: None, + lambda string, encoding='utf8', errors='strict': None], + compile=[ + (0, lambda source, filename, mode, flags=0, + dont_inherit=False, optimize=-1: None)], + max=[ + (1, lambda iterable: None, ('default', 'key',)), + (1, lambda arg1, arg2, *args: None, ('key',))], + min=[ + (1, lambda iterable: None, ('default', 'key',)), + (1, lambda arg1, arg2, *args: None, ('key',))], + open=[ + (0, lambda file, mode='r', buffering=-1, encoding=None, + errors=None, newline=None, closefd=True, opener=None: None)], + sorted=[ + (1, lambda iterable: None, ('key', 'reverse'))], + str=[ + lambda object='', encoding='utf', errors='strict': None], + ) + module_info[builtins]['print'] = [ + (0, lambda *args: None, ('sep', 'end', 'file', 'flush',))] + +else: # pragma: py3 no cover + module_info[builtins].update( + bytes=[ + lambda object='': None], + compile=[ + (0, lambda source, filename, mode, flags=0, + dont_inherit=False: None)], + max=[ + (1, lambda iterable, *args: None, ('key',))], + min=[ + (1, lambda iterable, *args: None, ('key',))], + open=[ + (0, lambda file, mode='r', buffering=-1: None)], + sorted=[ + lambda iterable, cmp=None, key=None, reverse=False: None], + str=[ + lambda object='': None], + ) + module_info[builtins]['print'] = [ + (0, lambda *args: None, ('sep', 'end', 'file',))] + +module_info[functools] = dict( + cmp_to_key=[ + (0, lambda mycmp: None)], + partial=[ + lambda func, *args, **kwargs: None], + partialmethod=[ + lambda func, *args, **kwargs: None], + reduce=[ + lambda function, sequence: None, + lambda function, sequence, initial: None], +) + +module_info[itertools] = dict( + accumulate=[ + (0, lambda iterable, func=None: None)], + chain=[ + lambda *iterables: None], + combinations=[ + (0, lambda iterable, r: None)], + combinations_with_replacement=[ + (0, lambda iterable, r: None)], + compress=[ + (0, lambda data, selectors: None)], + count=[ + lambda start=0, step=1: None], + cycle=[ + lambda iterable: None], + dropwhile=[ + lambda predicate, iterable: None], + filterfalse=[ + lambda function, sequence: None], + groupby=[ + (0, lambda iterable, key=None: None)], + ifilter=[ + lambda function, sequence: None], + ifilterfalse=[ + lambda function, sequence: None], + imap=[ + lambda func, sequence, *iterables: None], + islice=[ + lambda iterable, stop: None, + lambda iterable, start, stop: None, + lambda iterable, start, stop, step: None], + izip=[ + lambda *iterables: None], + izip_longest=[ + (0, lambda *iterables: None, ('fillvalue',))], + permutations=[ + (0, lambda iterable, r=0: None)], + repeat=[ + (0, lambda object, times=0: None)], + starmap=[ + lambda function, sequence: None], + takewhile=[ + lambda predicate, iterable: None], + tee=[ + lambda iterable: None, + lambda iterable, n: None], + zip_longest=[ + (0, lambda *iterables: None, ('fillvalue',))], +) + +if PY3: # pragma: py2 no cover + module_info[itertools].update( + product=[ + (0, lambda *iterables: None, ('repeat',))], + ) +else: # pragma: py3 no cover + module_info[itertools].update( + product=[ + lambda *iterables: None], + ) + +module_info[operator] = dict( + __abs__=[ + lambda a: None], + __add__=[ + lambda a, b: None], + __and__=[ + lambda a, b: None], + __concat__=[ + lambda a, b: None], + __contains__=[ + lambda a, b: None], + __delitem__=[ + lambda a, b: None], + __delslice__=[ + lambda a, b, c: None], + __div__=[ + lambda a, b: None], + __eq__=[ + lambda a, b: None], + __floordiv__=[ + lambda a, b: None], + __ge__=[ + lambda a, b: None], + __getitem__=[ + lambda a, b: None], + __getslice__=[ + lambda a, b, c: None], + __gt__=[ + lambda a, b: None], + __iadd__=[ + lambda a, b: None], + __iand__=[ + lambda a, b: None], + __iconcat__=[ + lambda a, b: None], + __idiv__=[ + lambda a, b: None], + __ifloordiv__=[ + lambda a, b: None], + __ilshift__=[ + lambda a, b: None], + __imatmul__=[ + lambda a, b: None], + __imod__=[ + lambda a, b: None], + __imul__=[ + lambda a, b: None], + __index__=[ + lambda a: None], + __inv__=[ + lambda a: None], + __invert__=[ + lambda a: None], + __ior__=[ + lambda a, b: None], + __ipow__=[ + lambda a, b: None], + __irepeat__=[ + lambda a, b: None], + __irshift__=[ + lambda a, b: None], + __isub__=[ + lambda a, b: None], + __itruediv__=[ + lambda a, b: None], + __ixor__=[ + lambda a, b: None], + __le__=[ + lambda a, b: None], + __lshift__=[ + lambda a, b: None], + __lt__=[ + lambda a, b: None], + __matmul__=[ + lambda a, b: None], + __mod__=[ + lambda a, b: None], + __mul__=[ + lambda a, b: None], + __ne__=[ + lambda a, b: None], + __neg__=[ + lambda a: None], + __not__=[ + lambda a: None], + __or__=[ + lambda a, b: None], + __pos__=[ + lambda a: None], + __pow__=[ + lambda a, b: None], + __repeat__=[ + lambda a, b: None], + __rshift__=[ + lambda a, b: None], + __setitem__=[ + lambda a, b, c: None], + __setslice__=[ + lambda a, b, c, d: None], + __sub__=[ + lambda a, b: None], + __truediv__=[ + lambda a, b: None], + __xor__=[ + lambda a, b: None], + _abs=[ + lambda x: None], + _compare_digest=[ + lambda a, b: None], + abs=[ + lambda a: None], + add=[ + lambda a, b: None], + and_=[ + lambda a, b: None], + attrgetter=[ + lambda attr, *args: None], + concat=[ + lambda a, b: None], + contains=[ + lambda a, b: None], + countOf=[ + lambda a, b: None], + delitem=[ + lambda a, b: None], + delslice=[ + lambda a, b, c: None], + div=[ + lambda a, b: None], + eq=[ + lambda a, b: None], + floordiv=[ + lambda a, b: None], + ge=[ + lambda a, b: None], + getitem=[ + lambda a, b: None], + getslice=[ + lambda a, b, c: None], + gt=[ + lambda a, b: None], + iadd=[ + lambda a, b: None], + iand=[ + lambda a, b: None], + iconcat=[ + lambda a, b: None], + idiv=[ + lambda a, b: None], + ifloordiv=[ + lambda a, b: None], + ilshift=[ + lambda a, b: None], + imatmul=[ + lambda a, b: None], + imod=[ + lambda a, b: None], + imul=[ + lambda a, b: None], + index=[ + lambda a: None], + indexOf=[ + lambda a, b: None], + inv=[ + lambda a: None], + invert=[ + lambda a: None], + ior=[ + lambda a, b: None], + ipow=[ + lambda a, b: None], + irepeat=[ + lambda a, b: None], + irshift=[ + lambda a, b: None], + is_=[ + lambda a, b: None], + is_not=[ + lambda a, b: None], + isCallable=[ + lambda a: None], + isMappingType=[ + lambda a: None], + isNumberType=[ + lambda a: None], + isSequenceType=[ + lambda a: None], + isub=[ + lambda a, b: None], + itemgetter=[ + lambda item, *args: None], + itruediv=[ + lambda a, b: None], + ixor=[ + lambda a, b: None], + le=[ + lambda a, b: None], + length_hint=[ + lambda obj: None, + lambda obj, default: None], + lshift=[ + lambda a, b: None], + lt=[ + lambda a, b: None], + matmul=[ + lambda a, b: None], + methodcaller=[ + lambda name, *args, **kwargs: None], + mod=[ + lambda a, b: None], + mul=[ + lambda a, b: None], + ne=[ + lambda a, b: None], + neg=[ + lambda a: None], + not_=[ + lambda a: None], + or_=[ + lambda a, b: None], + pos=[ + lambda a: None], + pow=[ + lambda a, b: None], + repeat=[ + lambda a, b: None], + rshift=[ + lambda a, b: None], + sequenceIncludes=[ + lambda a, b: None], + setitem=[ + lambda a, b, c: None], + setslice=[ + lambda a, b, c, d: None], + sub=[ + lambda a, b: None], + truediv=[ + lambda a, b: None], + truth=[ + lambda a: None], + xor=[ + lambda a, b: None], +) + +if PY3: # pragma: py2 no cover + def num_pos_args(func, sigspec): + """Return the number of positional arguments. ``f(x, y=1)`` has 1.""" + return sum(1 for x in sigspec.parameters.values() + if x.kind == x.POSITIONAL_OR_KEYWORD and + x.default is x.empty) + + def get_exclude_keywords(func, num_pos_only, sigspec): + """Return the names of position-only arguments if func has **kwargs""" + if num_pos_only == 0: + return () + has_kwargs = any(x.kind == x.VAR_KEYWORD + for x in sigspec.parameters.values()) + if not has_kwargs: + return () + pos_args = list(sigspec.parameters.values())[:num_pos_only] + return tuple(x.name for x in pos_args) + + def signature_or_spec(func): + try: + return inspect.signature(func) + except (ValueError, TypeError) as e: + return e + +else: # pragma: py3 no cover + def num_pos_args(func, sigspec): + """Return the number of positional arguments. ``f(x, y=1)`` has 1.""" + if sigspec.defaults: + return len(sigspec.args) - len(sigspec.defaults) + return len(sigspec.args) + + def get_exclude_keywords(func, num_pos_only, sigspec): + """Return the names of position-only arguments if func has **kwargs""" + if num_pos_only == 0: + return () + has_kwargs = sigspec.keywords is not None + if not has_kwargs: + return () + return tuple(sigspec.args[:num_pos_only]) + + def signature_or_spec(func): + try: + return inspect.getargspec(func) + except TypeError as e: + return e + + +def expand_sig(sig): + """Convert the signature spec in ``module_info`` to add to ``signatures``. + + The input signature spec is one of: + - ``lambda_func`` + - ``(num_position_args, lambda_func)`` + - ``(num_position_args, lambda_func, keyword_only_args)`` + + The output signature spec is: + ``(num_position_args, lambda_func, keyword_exclude, sigspec)`` + + where ``keyword_exclude`` includes keyword only arguments and, if variadic + keywords is present, the names of position-only argument. The latter is + included to support builtins such as ``partial(func, *args, **kwargs)``, + which allows ``func=`` to be used as a keyword even though it's the name + of a positional argument. + + """ + if isinstance(sig, tuple): + if len(sig) == 3: + num_pos_only, func, keyword_only = sig + assert isinstance(sig[-1], tuple) + else: + num_pos_only, func = sig + keyword_only = () + sigspec = signature_or_spec(func) + else: + func = sig + sigspec = signature_or_spec(func) + num_pos_only = num_pos_args(func, sigspec) + keyword_only = () + keyword_exclude = get_exclude_keywords(func, num_pos_only, sigspec) + return (num_pos_only, func, keyword_only + keyword_exclude, sigspec) + + +signatures = {} +for module, info in module_info.items(): + for name, sigs in info.items(): + if hasattr(module, name): + new_sigs = tuple(expand_sig(sig) for sig in sigs) + signatures[getattr(module, name)] = new_sigs + + +def check_valid(sig, args, kwargs): + """Like ``is_valid_args`` for the given signature spec.""" + num_pos_only, func, keyword_exclude, sigspec = sig + if len(args) < num_pos_only: + return False + if keyword_exclude: + kwargs = dict(kwargs) + for item in keyword_exclude: + kwargs.pop(item, None) + try: + func(*args, **kwargs) + return True + except TypeError: + return False + + +def check_partial(sig, args, kwargs): + """Like ``is_partial_args`` for the given signature spec.""" + num_pos_only, func, keyword_exclude, sigspec = sig + if len(args) < num_pos_only: + pad = (None,) * (num_pos_only - len(args)) + args = args + pad + if keyword_exclude: + kwargs = dict(kwargs) + for item in keyword_exclude: + kwargs.pop(item, None) + return is_partial_args(func, args, kwargs, sigspec=sigspec) + + +def is_builtin_valid_args(func, args, kwargs): + """Like ``is_valid_args`` for builtins in our ``signatures`` registry.""" + if func not in signatures: + return None + sigs = signatures[func] + return any(check_valid(sig, args, kwargs) for sig in sigs) + + +def is_builtin_partial_args(func, args, kwargs): + """Like ``is_partial_args`` for builtins in our ``signatures`` registry.""" + if func not in signatures: + return None + sigs = signatures[func] + return any(check_partial(sig, args, kwargs) for sig in sigs) + + +if PY3: # pragma: py2 no cover + def has_unknown_args(func, sigspec=None): + """Might ``func`` have ``*args`` that is passed to a wrapped function? + + This is specifically to support ``curry``. + + """ + if func in signatures: + return False + if sigspec is None: + try: + sigspec = inspect.signature(func) + except (ValueError, TypeError) as e: + sigspec = e + if isinstance(sigspec, ValueError): + return True + elif isinstance(sigspec, TypeError): + return False + try: + return any(x.kind == x.VAR_POSITIONAL + for x in sigspec.parameters.values()) + except AttributeError: # pragma: no cover + return False + +else: # pragma: py3 no cover + def has_unknown_args(func, sigspec=None): + """Might ``func`` have ``*args`` that is passed to a wrapped function? + + This is specifically to support ``curry``. + + """ + if func in signatures: + return False + if sigspec is None: + try: + sigspec = inspect.getargspec(func) + except TypeError as e: + sigspec = e + if isinstance(sigspec, TypeError): + return callable(func) + return sigspec.varargs is not None + + +from .functoolz import is_partial_args diff --git a/cytoolz/compatibility.py b/cytoolz/compatibility.py index 4360064..ad34a83 100644 --- a/cytoolz/compatibility.py +++ b/cytoolz/compatibility.py @@ -1,6 +1,7 @@ import operator import sys PY3 = sys.version_info[0] > 2 +PY34 = sys.version_info[0] == 3 and sys.version_info[1] == 4 __all__ = ['PY3', 'map', 'filter', 'range', 'zip', 'reduce', 'zip_longest', 'iteritems', 'iterkeys', 'itervalues'] diff --git a/cytoolz/curried/__init__.py b/cytoolz/curried/__init__.py index d10ff09..ecb71a7 100644 --- a/cytoolz/curried/__init__.py +++ b/cytoolz/curried/__init__.py @@ -76,6 +76,9 @@ cytoolz.update_in, cytoolz.keyfilter, cytoolz.groupby, + cytoolz.assoc_in, + cytoolz.random_sample, + cytoolz.excepts, ]) diff --git a/cytoolz/curried/exceptions.pyx b/cytoolz/curried/exceptions.pyx index 75ffe66..b95cdf6 100644 --- a/cytoolz/curried/exceptions.pyx +++ b/cytoolz/curried/exceptions.pyx @@ -1,4 +1,3 @@ -#cython: embedsignature=True from cytoolz import curry from cpython.dict cimport PyDict_Check diff --git a/cytoolz/dicttoolz.pxd b/cytoolz/dicttoolz.pxd index 4c99698..3133dde 100644 --- a/cytoolz/dicttoolz.pxd +++ b/cytoolz/dicttoolz.pxd @@ -35,6 +35,9 @@ cpdef object itemfilter(object predicate, object d, object factory=*) cpdef object assoc(object d, object key, object value, object factory=*) +cpdef object assoc_in(object d, object keys, object value, object factory=*) + + cdef object c_dissoc(object d, object keys) diff --git a/cytoolz/dicttoolz.pyx b/cytoolz/dicttoolz.pyx index 8814560..a06a585 100644 --- a/cytoolz/dicttoolz.pyx +++ b/cytoolz/dicttoolz.pyx @@ -1,4 +1,3 @@ -#cython: embedsignature=True from cpython.dict cimport (PyDict_Check, PyDict_CheckExact, PyDict_GetItem, PyDict_Merge, PyDict_New, PyDict_Next, PyDict_SetItem, PyDict_Update, PyDict_DelItem) @@ -15,7 +14,8 @@ from copy import copy __all__ = ['merge', 'merge_with', 'valmap', 'keymap', 'itemmap', 'valfilter', - 'keyfilter', 'itemfilter', 'assoc', 'dissoc', 'get_in', 'update_in'] + 'keyfilter', 'itemfilter', 'assoc', 'dissoc', 'assoc_in', 'get_in', + 'update_in'] cdef int PyMapping_Next(object p, Py_ssize_t *ppos, PyObject* *pkey, PyObject* *pval) except -1: @@ -374,11 +374,54 @@ cpdef object assoc(object d, object key, object value, object factory=dict): return rv +cpdef object assoc_in(object d, object keys, object value, object factory=dict): + """ + Return a new dict with new, potentially nested, key value pair + + >>> purchase = {'name': 'Alice', + ... 'order': {'items': ['Apple', 'Orange'], + ... 'costs': [0.50, 1.25]}, + ... 'credit card': '5555-1234-1234-1234'} + >>> assoc_in(purchase, ['order', 'costs'], [0.25, 1.00]) # doctest: +SKIP + {'credit card': '5555-1234-1234-1234', + 'name': 'Alice', + 'purchase': {'costs': [0.25, 1.00], 'items': ['Apple', 'Orange']}} + """ + cdef object prevkey, key + cdef object rv, inner, dtemp + prevkey, keys = keys[0], keys[1:] + rv = factory() + if PyDict_CheckExact(rv): + PyDict_Update(rv, d) + else: + rv.update(d) + inner = rv + + for key in keys: + if prevkey in d: + d = d[prevkey] + dtemp = factory() + if PyDict_CheckExact(dtemp): + PyDict_Update(dtemp, d) + else: + dtemp.update(d) + else: + d = factory() + dtemp = d + inner[prevkey] = dtemp + prevkey = key + inner = dtemp + + inner[prevkey] = value + return rv + + cdef object c_dissoc(object d, object keys): cdef object rv, key rv = copy(d) for key in keys: - del rv[key] + if key in rv: + del rv[key] return rv diff --git a/cytoolz/functoolz.pxd b/cytoolz/functoolz.pxd index a0d9f13..0c8e871 100644 --- a/cytoolz/functoolz.pxd +++ b/cytoolz/functoolz.pxd @@ -8,6 +8,8 @@ cdef object c_thread_last(object val, object forms) cdef class curry: + cdef readonly object _sigspec + cdef readonly object _has_unknown_args cdef readonly object func cdef readonly tuple args cdef readonly dict keywords @@ -48,3 +50,18 @@ cdef object c_juxt(object funcs) cpdef object do(object func, object x) + + +cpdef object return_none(object exc) + + +cdef class excepts: + cdef public object exc + cdef public object func + cdef public object handler + + +cpdef object is_valid_args(object func, object args, object kwargs, object sigspec=*) + + +cpdef object is_partial_args(object func, object args, object kwargs, object sigspec=*) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index aadeefb..6b3ecfa 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -1,9 +1,15 @@ -#cython: embedsignature=True import inspect import sys from functools import partial -from cytoolz.compatibility import filter as ifilter, map as imap, reduce - +from operator import attrgetter +from textwrap import dedent +from cytoolz.compatibility import PY3, PY34, filter as ifilter, map as imap, reduce +from cytoolz._signatures import ( + is_builtin_valid_args as _is_builtin_valid_args, + is_builtin_partial_args as _is_builtin_partial_args, + has_unknown_args as _has_unknown_args, + signature_or_spec as _signature_or_spec, +) from cpython.dict cimport PyDict_Merge, PyDict_New from cpython.exc cimport PyErr_Clear, PyErr_Occurred, PyErr_GivenExceptionMatches from cpython.object cimport (PyCallable_Check, PyObject_Call, PyObject_CallObject, @@ -18,7 +24,8 @@ from cytoolz.cpython cimport PtrObject_Call __all__ = ['identity', 'thread_first', 'thread_last', 'memoize', 'compose', - 'pipe', 'complement', 'juxt', 'do', 'curry', 'memoize', 'flip'] + 'pipe', 'complement', 'juxt', 'do', 'curry', 'memoize', 'flip', + 'excepts'] cpdef object identity(object x): @@ -114,44 +121,6 @@ def thread_last(val, *forms): return c_thread_last(val, forms) -# This is a kludge for Python 3.4.0 support -# currently len(inspect.getargspec(map).args) == 0, a wrong result. -# As this is fixed in future versions then hopefully this kludge can be -# removed. -known_numargs = {map: 2, filter: 2, reduce: 2, imap: 2, ifilter: 2} - - -cpdef Py_ssize_t _num_required_args(object func) except *: - """ - Number of args for func - - >>> def foo(a, b, c=None): - ... return a + b + c - - >>> _num_required_args(foo) - 2 - - >>> def bar(*args): - ... return sum(args) - - >>> print(_num_required_args(bar)) - -1 - """ - cdef Py_ssize_t num_defaults - - if func in known_numargs: - return known_numargs[func] - try: - spec = inspect.getargspec(func) - if spec.varargs: - return -1 - num_defaults = len(spec.defaults) if spec.defaults else 0 - return len(spec.args) - num_defaults - except TypeError: - pass - return -1 - - cdef struct partialobject: PyObject _ PyObject *fn @@ -200,6 +169,10 @@ cdef class curry: cytoolz.curried - namespace of curried functions http://toolz.readthedocs.org/en/latest/curry.html """ + property __wrapped__: + def __get__(self): + return self.func + def __cinit__(self, *args, **kwargs): if not args: raise TypeError('__init__() takes at least 2 arguments (1 given)') @@ -226,6 +199,8 @@ cdef class curry: self.keywords = kwargs if kwargs else _empty_kwargs() self.__doc__ = getattr(func, '__doc__', None) self.__name__ = getattr(func, '__name__', '') + self._sigspec = None + self._has_unknown_args = None def __str__(self): return str(self.func) @@ -249,7 +224,6 @@ cdef class curry: def __call__(self, *args, **kwargs): cdef PyObject *obj - cdef Py_ssize_t required_args cdef object val if PyTuple_GET_SIZE(args) == 0: @@ -267,20 +241,72 @@ cdef class curry: val = PyErr_Occurred() PyErr_Clear() - if PyErr_GivenExceptionMatches(val, TypeError): - required_args = _num_required_args(self.func) - # If there was a genuine TypeError - if required_args == -1 or len(args) < required_args: - return curry(self.func, *args, **kwargs) + if (PyErr_GivenExceptionMatches(val, TypeError) and + self._should_curry_internal(args, kwargs, val) + ): + return type(self)(self.func, *args, **kwargs) + raise val + + def _should_curry_internal(self, args, kwargs, exc=None): + func = self.func + + # `toolz` has these three lines + #args = self.args + args + #if self.keywords: + # kwargs = dict(self.keywords, **kwargs) + + if self._sigspec is None: + sigspec = self._sigspec = _signature_or_spec(func) + self._has_unknown_args = _has_unknown_args(func, sigspec=sigspec) + else: + sigspec = self._sigspec + + if is_partial_args(func, args, kwargs, sigspec=sigspec) is False: + # Nothing can make the call valid + return False + elif self._has_unknown_args: + # The call may be valid and raised a TypeError, but we curry + # anyway because the function may have `*args`. This is useful + # for decorators with signature `func(*args, **kwargs)`. + return True + elif not is_valid_args(func, args, kwargs, sigspec=sigspec): + # Adding more arguments may make the call valid + return True + else: + # There was a genuine TypeError + return False + + def bind(self, *args, **kwargs): + return type(self)(self, *args, **kwargs) + + def call(self, *args, **kwargs): + cdef PyObject *obj + cdef object val + + if PyTuple_GET_SIZE(args) == 0: + args = self.args + elif PyTuple_GET_SIZE(self.args) != 0: + args = PySequence_Concat(self.args, args) + if self.keywords is not None: + PyDict_Merge(kwargs, self.keywords, False) + + obj = PtrObject_Call(self.func, args, kwargs) + if obj is not NULL: + val = obj + Py_DECREF(val) + return val + + val = PyErr_Occurred() + PyErr_Clear() raise val def __get__(self, instance, owner): if instance is None: return self - return curry(self, instance) + return type(self)(self, instance) def __reduce__(self): - return (curry, (self.func,), (self.args, self.keywords)) + return (type(self), (self.func,), (self.args, self.keywords)) def __setstate__(self, state): self.args, self.keywords = state @@ -338,6 +364,10 @@ cdef class c_memoize: def __get__(self): return self.func.__name__ + property __wrapped__: + def __get__(self): + return self.func + def __cinit__(self, func, cache=None, key=None): self.func = func if cache is None: @@ -635,3 +665,229 @@ cpdef object _flip(object f, object a, object b): flip = curry(_flip) + + +cpdef object return_none(object exc): + """ + Returns None. + """ + return None + + +cdef class excepts: + """ + A wrapper around a function to catch exceptions and + dispatch to a handler. + + This is like a functional try/except block, in the same way that + ifexprs are functional if/else blocks. + + Examples + -------- + >>> excepting = excepts( + ... ValueError, + ... lambda a: [1, 2].index(a), + ... lambda _: -1, + ... ) + >>> excepting(1) + 0 + >>> excepting(3) + -1 + + Multiple exceptions and default except clause. + >>> excepting = excepts((IndexError, KeyError), lambda a: a[0]) + >>> excepting([]) + >>> excepting([1]) + 1 + >>> excepting({}) + >>> excepting({0: 1}) + 1 + """ + + def __init__(self, exc, func, handler=return_none): + self.exc = exc + self.func = func + self.handler = handler + + def __call__(self, *args, **kwargs): + try: + return self.func(*args, **kwargs) + except self.exc as e: + return self.handler(e) + + property __name__: + def __get__(self): + exc = self.exc + try: + if isinstance(exc, tuple): + exc_name = '_or_'.join(map(attrgetter('__name__'), exc)) + else: + exc_name = exc.__name__ + return '%s_excepting_%s' % (self.func.__name__, exc_name) + except AttributeError: + return 'excepting' + + property __doc__: + def __get__(self): + exc = self.exc + try: + if isinstance(exc, tuple): + exc_name = '(%s)' % ', '.join( + map(attrgetter('__name__'), exc), + ) + else: + exc_name = exc.__name__ + + return dedent( + """\ + A wrapper around {inst.func.__name__!r} that will except: + {exc} + and handle any exceptions with {inst.handler.__name__!r}. + + Docs for {inst.func.__name__!r}: + {inst.func.__doc__} + + Docs for {inst.handler.__name__!r}: + {inst.handler.__doc__} + """ + ).format( + inst=self, + exc=exc_name, + ) + except AttributeError: + return type(self).__doc__ + + +cpdef object is_valid_args(object func, object args, object kwargs, object sigspec=None): + if PY34: + val = _is_builtin_valid_args(func, args, kwargs) + if val is not None: + return val + if PY3: + if sigspec is None: + try: + sigspec = inspect.signature(func) + except (ValueError, TypeError) as e: + sigspec = e + if isinstance(sigspec, ValueError): + return _is_builtin_valid_args(func, args, kwargs) + elif isinstance(sigspec, TypeError): + return False + try: + sigspec.bind(*args, **kwargs) + except (TypeError, AttributeError): + return False + return True + + else: + if sigspec is None: + try: + sigspec = inspect.getargspec(func) + except TypeError as e: + sigspec = e + if isinstance(sigspec, TypeError): + if not callable(func): + return False + return _is_builtin_valid_args(func, args, kwargs) + + spec = sigspec + defaults = spec.defaults or () + num_pos = len(spec.args) - len(defaults) + missing_pos = spec.args[len(args):num_pos] + for arg in missing_pos: + if arg not in kwargs: + return False + + if spec.varargs is None: + num_extra_pos = max(0, len(args) - num_pos) + else: + num_extra_pos = 0 + + kwargs = dict(kwargs) + + # Add missing keyword arguments (unless already included in `args`) + missing_kwargs = spec.args[num_pos + num_extra_pos:] + kwargs.update(zip(missing_kwargs, defaults[num_extra_pos:])) + + # Convert call to use positional arguments + more_args = [] + for key in spec.args[len(args):]: + more_args.append(kwargs.pop(key)) + args = args + tuple(more_args) + + if ( + not spec.keywords and kwargs or + not spec.varargs and len(args) > len(spec.args) or + set(spec.args[:len(args)]) & set(kwargs) + ): + return False + else: + return True + + +cpdef object is_partial_args(object func, object args, object kwargs, object sigspec=None): + if PY34: + val = _is_builtin_partial_args(func, args, kwargs) + if val is not None: + return val + if PY3: + if sigspec is None: + try: + sigspec = inspect.signature(func) + except (ValueError, TypeError) as e: + sigspec = e + if isinstance(sigspec, ValueError): + return _is_builtin_partial_args(func, args, kwargs) + elif isinstance(sigspec, TypeError): + return False + try: + sigspec.bind_partial(*args, **kwargs) + except (TypeError, AttributeError): + return False + return True + + else: + if sigspec is None: + try: + sigspec = inspect.getargspec(func) + except TypeError as e: + sigspec = e + if isinstance(sigspec, TypeError): + if not callable(func): + return False + return _is_builtin_partial_args(func, args, kwargs) + + spec = sigspec + defaults = spec.defaults or () + num_pos = len(spec.args) - len(defaults) + if spec.varargs is None: + num_extra_pos = max(0, len(args) - num_pos) + else: + num_extra_pos = 0 + + kwargs = dict(kwargs) + + # Add missing keyword arguments (unless already included in `args`) + missing_kwargs = spec.args[num_pos + num_extra_pos:] + kwargs.update(zip(missing_kwargs, defaults[num_extra_pos:])) + + # Add missing position arguments as keywords (may already be in kwargs) + missing_args = spec.args[len(args):num_pos + num_extra_pos] + for x in missing_args: + kwargs[x] = None + + # Convert call to use positional arguments + more_args = [] + for key in spec.args[len(args):]: + more_args.append(kwargs.pop(key)) + args = args + tuple(more_args) + + if ( + not spec.keywords and kwargs or + not spec.varargs and len(args) > len(spec.args) or + set(spec.args[:len(args)]) & set(kwargs) + ): + return False + else: + return True + diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index 9ddb1d6..900787f 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -258,3 +258,9 @@ cpdef object topk(Py_ssize_t k, object seq, object key=*) cpdef object peek(object seq) + + +cdef class random_sample: + cdef object iter_seq + cdef object prob + cdef object random_func diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 575c72b..56ec276 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -1,4 +1,3 @@ -#cython: embedsignature=True from cpython.dict cimport PyDict_GetItem, PyDict_SetItem from cpython.exc cimport PyErr_Clear, PyErr_GivenExceptionMatches, PyErr_Occurred from cpython.list cimport PyList_Append, PyList_GET_ITEM, PyList_GET_SIZE @@ -15,6 +14,7 @@ from collections import deque from heapq import heapify, heappop, heapreplace from itertools import chain, islice from operator import itemgetter +from random import Random from cytoolz.compatibility import map, zip, zip_longest from cytoolz.utils import no_default @@ -24,7 +24,7 @@ __all__ = ['remove', 'accumulate', 'groupby', 'merge_sorted', 'interleave', 'first', 'second', 'nth', 'last', 'get', 'concat', 'concatv', 'mapcat', 'cons', 'interpose', 'frequencies', 'reduceby', 'iterate', 'sliding_window', 'partition', 'partition_all', 'count', 'pluck', - 'join', 'tail', 'diff', 'topk', 'peek'] + 'join', 'tail', 'diff', 'topk', 'peek', 'random_sample'] concatv = chain @@ -1688,3 +1688,55 @@ cpdef object peek(object seq): iterator = iter(seq) item = next(iterator) return item, chain((item,), iterator) + + +cdef class random_sample: + """ random_sample(prob, seq, random_state=None) + + Return elements from a sequence with probability of prob + + Returns a lazy iterator of random items from seq. + + ``random_sample`` considers each item independently and without + replacement. See below how the first time it returned 13 items and the + next time it returned 6 items. + + >>> seq = list(range(100)) + >>> list(random_sample(0.1, seq)) # doctest: +SKIP + [6, 9, 19, 35, 45, 50, 58, 62, 68, 72, 78, 86, 95] + >>> list(random_sample(0.1, seq)) # doctest: +SKIP + [6, 44, 54, 61, 69, 94] + + Providing an integer seed for ``random_state`` will result in + deterministic sampling. Given the same seed it will return the same sample + every time. + + >>> list(random_sample(0.1, seq, random_state=2016)) + [7, 9, 19, 25, 30, 32, 34, 48, 59, 60, 81, 98] + >>> list(random_sample(0.1, seq, random_state=2016)) + [7, 9, 19, 25, 30, 32, 34, 48, 59, 60, 81, 98] + + ``random_state`` can also be any object with a method ``random`` that + returns floats between 0.0 and 1.0 (exclusive). + + >>> from random import Random + >>> randobj = Random(2016) + >>> list(random_sample(0.1, seq, random_state=randobj)) + [7, 9, 19, 25, 30, 32, 34, 48, 59, 60, 81, 98] + """ + def __cinit__(self, object prob, object seq, random_state=None): + float(prob) + self.prob = prob + self.iter_seq = iter(seq) + if not hasattr(random_state, 'random'): + random_state = Random(random_state) + self.random_func = random_state.random + + def __iter__(self): + return self + + def __next__(self): + while True: + if self.random_func() < self.prob: + return next(self.iter_seq) + next(self.iter_seq) diff --git a/cytoolz/recipes.pyx b/cytoolz/recipes.pyx index 1a74b60..e6b5b91 100644 --- a/cytoolz/recipes.pyx +++ b/cytoolz/recipes.pyx @@ -1,4 +1,3 @@ -#cython: embedsignature=True from cpython.sequence cimport PySequence_Tuple from cytoolz.itertoolz cimport frequencies, pluck diff --git a/cytoolz/tests/test_dicttoolz.py b/cytoolz/tests/test_dicttoolz.py index 3c213fd..6327631 100644 --- a/cytoolz/tests/test_dicttoolz.py +++ b/cytoolz/tests/test_dicttoolz.py @@ -1,7 +1,7 @@ from collections import defaultdict as _defaultdict from cytoolz.dicttoolz import (merge, merge_with, valmap, keymap, update_in, assoc, dissoc, keyfilter, valfilter, itemmap, - itemfilter) + itemfilter, assoc_in) from cytoolz.utils import raises from cytoolz.compatibility import PY3 @@ -94,6 +94,7 @@ def test_dissoc(self): assert dissoc(D({"a": 1, "b": 2}), "a") == D({"b": 2}) assert dissoc(D({"a": 1, "b": 2}), "b") == D({"a": 1}) assert dissoc(D({"a": 1, "b": 2}), "a", "b") == D({}) + assert dissoc(D({"a": 1}), "a") == dissoc(dissoc(D({"a": 1}), "a"), "a") # Verify immutability: d = D({'x': 1}) @@ -102,6 +103,20 @@ def test_dissoc(self): assert d is oldd assert d2 is not oldd + def test_assoc_in(self): + D, kw = self.D, self.kw + assert assoc_in(D({"a": 1}), ["a"], 2, **kw) == D({"a": 2}) + assert (assoc_in(D({"a": D({"b": 1})}), ["a", "b"], 2, **kw) == + D({"a": D({"b": 2})})) + assert assoc_in(D({}), ["a", "b"], 1, **kw) == D({"a": D({"b": 1})}) + + # Verify immutability: + d = D({'x': 1}) + oldd = d + d2 = assoc_in(d, ['x'], 2, **kw) + assert d is oldd + assert d2 is not oldd + def test_update_in(self): D, kw = self.D, self.kw assert update_in(D({"a": 0}), ["a"], inc, **kw) == D({"a": 1}) @@ -235,4 +250,3 @@ class TestCustomMapping(TestDict): """ D = CustomMapping kw = {'factory': lambda: CustomMapping()} - diff --git a/cytoolz/tests/test_embedded_sigs.py b/cytoolz/tests/test_embedded_sigs.py index 1d037c2..2046697 100644 --- a/cytoolz/tests/test_embedded_sigs.py +++ b/cytoolz/tests/test_embedded_sigs.py @@ -32,6 +32,8 @@ def test_class_sigs(): d = merge_with(identity, toolz_dict, cytoolz_dict) for key, (toolz_func, cytoolz_func) in d.items(): + if key in ['excepts', 'juxt']: + continue try: # function toolz_spec = inspect.getargspec(toolz_func) diff --git a/cytoolz/tests/test_functoolz.py b/cytoolz/tests/test_functoolz.py index 3e278fd..087cd8b 100644 --- a/cytoolz/tests/test_functoolz.py +++ b/cytoolz/tests/test_functoolz.py @@ -1,13 +1,11 @@ import platform - from cytoolz.functoolz import (thread_first, thread_last, memoize, curry, - compose, pipe, complement, do, juxt, flip) -from cytoolz.functoolz import _num_required_args + compose, pipe, complement, do, juxt, flip, excepts) from operator import add, mul, itemgetter from cytoolz.utils import raises from functools import partial - +from cytoolz.compatibility import PY3 def iseven(x): return x % 2 == 0 @@ -48,7 +46,7 @@ def f(x, y): return x + y mf = memoize(f) - assert mf(2, 3) == mf(2, 3) + assert mf(2, 3) is mf(2, 3) assert fn_calls == [1] # function was only called once assert mf.__doc__ == f.__doc__ assert raises(TypeError, lambda: mf(1, {})) @@ -153,6 +151,17 @@ def f(x, y, *args, **kwargs): assert f(1, 3) == 3 +def test_memoize_wrapped(): + + def foo(): + """ + Docstring + """ + pass + memoized_foo = memoize(foo) + assert memoized_foo.__wrapped__ is foo + + def test_curry_simple(): cmul = curry(mul) double = cmul(2) @@ -413,14 +422,79 @@ def __hash__(self): assert A.addstatic(3, 4) == 7 -def test__num_required_args(): - assert _num_required_args(map) != 0 - assert _num_required_args(lambda x: x) == 1 - assert _num_required_args(lambda x, y: x) == 2 +def test_curry_wrapped(): + + def foo(a): + """ + Docstring + """ + pass + curried_foo = curry(foo) + assert curried_foo.__wrapped__ is foo + + +def test_curry_call(): + @curry + def add(x, y): + return x + y + assert raises(TypeError, lambda: add.call(1)) + assert add(1)(2) == add.call(1, 2) + assert add(1)(2) == add(1).call(2) + + +def test_curry_bind(): + @curry + def add(x=1, y=2): + return x + y + assert add() == add(1, 2) + assert add.bind(10)(20) == add(10, 20) + assert add.bind(10).bind(20)() == add(10, 20) + assert add.bind(x=10)(y=20) == add(10, 20) + assert add.bind(x=10).bind(y=20)() == add(10, 20) + + +def test_curry_unknown_args(): + def add3(x, y, z): + return x + y + z + + @curry + def f(*args): + return add3(*args) + + assert f()(1)(2)(3) == 6 + assert f(1)(2)(3) == 6 + assert f(1, 2)(3) == 6 + assert f(1, 2, 3) == 6 + assert f(1, 2)(3, 4) == f(1, 2, 3, 4) + + +def test_curry_bad_types(): + assert raises(TypeError, lambda: curry(1)) - def foo(x, y, z=2): + +def test_curry_subclassable(): + class mycurry(curry): pass - assert _num_required_args(foo) == 2 + + add = mycurry(lambda x, y: x+y) + assert isinstance(add, curry) + assert isinstance(add, mycurry) + assert isinstance(add(1), mycurry) + assert isinstance(add()(1), mycurry) + assert add(1)(2) == 3 + + # Should we make `_should_curry` public? + """ + class curry2(curry): + def _should_curry(self, args, kwargs, exc=None): + return len(self.args) + len(args) < 2 + + add = curry2(lambda x, y: x+y) + assert isinstance(add(1), curry2) + assert add(1)(2) == 3 + assert isinstance(add(1)(x=2), curry2) + assert raises(TypeError, lambda: add(1)(x=2)(3)) + """ def test_compose(): @@ -508,3 +582,65 @@ def f(a, b): return a, b assert flip(f, 'a', 'b') == ('b', 'a') + + +def test_excepts(): + # These are descriptors, make sure this works correctly. + assert excepts.__name__ == 'excepts' + assert ( + 'A wrapper around a function to catch exceptions and\n' + ' dispatch to a handler.\n' + ) in excepts.__doc__ + + def idx(a): + """idx docstring + """ + return [1, 2].index(a) + + def handler(e): + """handler docstring + """ + assert isinstance(e, ValueError) + return -1 + + excepting = excepts(ValueError, idx, handler) + assert excepting(1) == 0 + assert excepting(2) == 1 + assert excepting(3) == -1 + + assert excepting.__name__ == 'idx_excepting_ValueError' + assert 'idx docstring' in excepting.__doc__ + assert 'ValueError' in excepting.__doc__ + assert 'handler docstring' in excepting.__doc__ + + def getzero(a): + """getzero docstring + """ + return a[0] + + excepting = excepts((IndexError, KeyError), getzero) + assert excepting([]) is None + assert excepting([1]) == 1 + assert excepting({}) is None + assert excepting({0: 1}) == 1 + + assert excepting.__name__ == 'getzero_excepting_IndexError_or_KeyError' + assert 'getzero docstring' in excepting.__doc__ + assert 'return_none' in excepting.__doc__ + assert 'Returns None' in excepting.__doc__ + + def raise_(a): + """A function that raises an instance of the exception type given. + """ + raise a() + + excepting = excepts((ValueError, KeyError), raise_) + assert excepting(ValueError) is None + assert excepting(KeyError) is None + assert raises(TypeError, lambda: excepting(TypeError)) + assert raises(NotImplementedError, lambda: excepting(NotImplementedError)) + + excepting = excepts(object(), object(), object()) + assert excepting.__name__ == 'excepting' + assert excepting.__doc__ == excepts.__doc__ + diff --git a/cytoolz/tests/test_inspect_args.py b/cytoolz/tests/test_inspect_args.py new file mode 100644 index 0000000..881d478 --- /dev/null +++ b/cytoolz/tests/test_inspect_args.py @@ -0,0 +1,267 @@ +import functools +import sys +from cytoolz.functoolz import curry, is_valid_args, is_partial_args +from cytoolz._signatures import has_unknown_args +from cytoolz.compatibility import PY3 +from cytoolz.utils import raises + + +def make_func(param_string, raise_if_called=True): + if not param_string.startswith('('): + param_string = '(%s)' % param_string + if raise_if_called: + body = 'raise ValueError("function should not be called")' + else: + body = 'return True' + d = {} + exec('def func%s:\n %s' % (param_string, body), globals(), d) + return d['func'] + + +def test_make_func(): + f = make_func('') + assert raises(ValueError, lambda: f()) + assert raises(TypeError, lambda: f(1)) + + f = make_func('', raise_if_called=False) + assert f() + assert raises(TypeError, lambda: f(1)) + + f = make_func('x, y=1', raise_if_called=False) + assert f(1) + assert f(x=1) + assert f(1, 2) + assert f(x=1, y=2) + assert raises(TypeError, lambda: f(1, 2, 3)) + + f = make_func('(x, y=1)', raise_if_called=False) + assert f(1) + assert f(x=1) + assert f(1, 2) + assert f(x=1, y=2) + assert raises(TypeError, lambda: f(1, 2, 3)) + + +def test_is_valid(check_valid=is_valid_args, incomplete=False): + orig_check_valid = check_valid + check_valid = lambda func, *args, **kwargs: orig_check_valid(func, args, kwargs) + + f = make_func('') + assert check_valid(f) + assert check_valid(f, 1) is False + assert check_valid(f, x=1) is False + + f = make_func('x') + assert check_valid(f) is incomplete + assert check_valid(f, 1) + assert check_valid(f, x=1) + assert check_valid(f, 1, x=2) is False + assert check_valid(f, 1, y=2) is False + assert check_valid(f, 1, 2) is False + assert check_valid(f, x=1, y=2) is False + + f = make_func('x=1') + assert check_valid(f) + assert check_valid(f, 1) + assert check_valid(f, x=1) + assert check_valid(f, 1, x=2) is False + assert check_valid(f, 1, y=2) is False + assert check_valid(f, 1, 2) is False + assert check_valid(f, x=1, y=2) is False + + f = make_func('*args') + assert check_valid(f) + assert check_valid(f, 1) + assert check_valid(f, 1, 2) + assert check_valid(f, x=1) is False + + f = make_func('**kwargs') + assert check_valid(f) + assert check_valid(f, x=1) + assert check_valid(f, x=1, y=2) + assert check_valid(f, 1) is False + + f = make_func('x, *args') + assert check_valid(f) is incomplete + assert check_valid(f, 1) + assert check_valid(f, 1, 2) + assert check_valid(f, x=1) + assert check_valid(f, 1, x=1) is False + assert check_valid(f, 1, y=1) is False + + f = make_func('x, y=1, **kwargs') + assert check_valid(f) is incomplete + assert check_valid(f, 1) + assert check_valid(f, x=1) + assert check_valid(f, 1, 2) + assert check_valid(f, x=1, y=2, z=3) + assert check_valid(f, 1, 2, y=3) is False + + f = make_func('a, b, c=3, d=4') + assert check_valid(f) is incomplete + assert check_valid(f, 1) is incomplete + assert check_valid(f, 1, 2) + assert check_valid(f, 1, c=3) is incomplete + assert check_valid(f, 1, e=3) is False + assert check_valid(f, 1, 2, e=3) is False + assert check_valid(f, 1, 2, b=3) is False + + assert check_valid(1) is False + + +def test_is_valid_py3(check_valid=is_valid_args, incomplete=False): + if not PY3: + return + orig_check_valid = check_valid + check_valid = lambda func, *args, **kwargs: orig_check_valid(func, args, kwargs) + + f = make_func('x, *, y=1') + assert check_valid(f) is incomplete + assert check_valid(f, 1) + assert check_valid(f, x=1) + assert check_valid(f, 1, y=2) + assert check_valid(f, 1, 2) is False + assert check_valid(f, 1, z=2) is False + + f = make_func('x, *args, y=1') + assert check_valid(f) is incomplete + assert check_valid(f, 1) + assert check_valid(f, x=1) + assert check_valid(f, 1, y=2) + assert check_valid(f, 1, 2, y=2) + assert check_valid(f, 1, 2) + assert check_valid(f, 1, z=2) is False + + f = make_func('*, y=1') + assert check_valid(f) + assert check_valid(f, 1) is False + assert check_valid(f, y=1) + assert check_valid(f, z=1) is False + + f = make_func('x, *, y') + assert check_valid(f) is incomplete + assert check_valid(f, 1) is incomplete + assert check_valid(f, x=1) is incomplete + assert check_valid(f, 1, y=2) + assert check_valid(f, x=1, y=2) + assert check_valid(f, 1, 2) is False + assert check_valid(f, 1, z=2) is False + assert check_valid(f, 1, y=1, z=2) is False + + f = make_func('x=1, *, y, z=3') + assert check_valid(f) is incomplete + assert check_valid(f, 1, z=3) is incomplete + assert check_valid(f, y=2) + assert check_valid(f, 1, y=2) + assert check_valid(f, x=1, y=2) + assert check_valid(f, x=1, y=2, z=3) + assert check_valid(f, 1, x=1, y=2) is False + assert check_valid(f, 1, 3, y=2) is False + + f = make_func('w, x=2, *args, y, z=4') + assert check_valid(f) is incomplete + assert check_valid(f, 1) is incomplete + assert check_valid(f, 1, y=3) + + f = make_func('a, b, c=3, d=4, *args, e=5, f=6, g, h') + assert check_valid(f) is incomplete + assert check_valid(f, 1) is incomplete + assert check_valid(f, 1, 2) is incomplete + assert check_valid(f, 1, 2, g=7) is incomplete + assert check_valid(f, 1, 2, g=7, h=8) + assert check_valid(f, 1, 2, 3, 4, 5, 6, 7, 8, 9) is incomplete + + f = make_func('a: int, b: float') + assert check_valid(f) is incomplete + assert check_valid(f, 1) is incomplete + assert check_valid(f, b=1) is incomplete + assert check_valid(f, 1, 2) + + f = make_func('(a: int, b: float) -> float') + assert check_valid(f) is incomplete + assert check_valid(f, 1) is incomplete + assert check_valid(f, b=1) is incomplete + assert check_valid(f, 1, 2) + + f.__signature__ = 34 + assert check_valid(f) is False + + class RaisesValueError(object): + def __call__(self): + pass + @property + def __signature__(self): + raise ValueError('Testing Python 3.4') + + f = RaisesValueError() + assert check_valid(f) is None + + +def test_is_partial(): + test_is_valid(check_valid=is_partial_args, incomplete=True) + test_is_valid_py3(check_valid=is_partial_args, incomplete=True) + + +def test_is_valid_curry(): + def check_curry(func, args, kwargs, incomplete=True): + try: + curry(func)(*args, **kwargs) + curry(func, *args)(**kwargs) + curry(func, **kwargs)(*args) + curry(func, *args, **kwargs)() + if not isinstance(func, type(lambda: None)): + return None + return incomplete + except ValueError: + return True + except TypeError: + return False + + check_valid = functools.partial(check_curry, incomplete=True) + test_is_valid(check_valid=check_valid, incomplete=True) + test_is_valid_py3(check_valid=check_valid, incomplete=True) + + check_valid = functools.partial(check_curry, incomplete=False) + test_is_valid(check_valid=check_valid, incomplete=False) + test_is_valid_py3(check_valid=check_valid, incomplete=False) + + +def test_func_keyword(): + def f(func=None): + pass + assert is_valid_args(f, (), {}) + assert is_valid_args(f, (None,), {}) + assert is_valid_args(f, (), {'func': None}) + assert is_valid_args(f, (None,), {'func': None}) is False + assert is_partial_args(f, (), {}) + assert is_partial_args(f, (None,), {}) + assert is_partial_args(f, (), {'func': None}) + assert is_partial_args(f, (None,), {'func': None}) is False + + +def test_has_unknown_args(): + assert has_unknown_args(1) is False + assert has_unknown_args(map) is False + assert has_unknown_args(make_func('')) is False + assert has_unknown_args(make_func('x, y, z')) is False + assert has_unknown_args(make_func('*args')) + assert has_unknown_args(make_func('**kwargs')) is False + assert has_unknown_args(make_func('x, y, *args, **kwargs')) + assert has_unknown_args(make_func('x, y, z=1')) is False + assert has_unknown_args(make_func('x, y, z=1, **kwargs')) is False + + if PY3: + f = make_func('*args') + f.__signature__ = 34 + assert has_unknown_args(f) is False + + class RaisesValueError(object): + def __call__(self): + pass + @property + def __signature__(self): + raise ValueError('Testing Python 3.4') + + f = RaisesValueError() + assert has_unknown_args(f) + diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index 84ca99f..c96e911 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -2,6 +2,7 @@ from itertools import starmap from cytoolz.utils import raises from functools import partial +from random import Random from cytoolz.itertoolz import (remove, groupby, merge_sorted, concat, concatv, interleave, unique, isiterable, getter, @@ -11,7 +12,7 @@ reduceby, iterate, accumulate, sliding_window, count, partition, partition_all, take_nth, pluck, join, - diff, topk, peek) + diff, topk, peek, random_sample) from cytoolz.compatibility import range, filter from operator import add, mul @@ -461,6 +462,8 @@ def test_topk(): assert topk(2, [{'a': 1, 'b': 10}, {'a': 2, 'b': 9}, {'a': 10, 'b': 1}, {'a': 9, 'b': 2}], key='b') == \ ({'a': 1, 'b': 10}, {'a': 2, 'b': 9}) + assert topk(2, [(0, 4), (1, 3), (2, 2), (3, 1), (4, 0)], 0) == \ + ((4, 0), (3, 1)) def test_topk_is_stable(): @@ -469,8 +472,32 @@ def test_topk_is_stable(): def test_peek(): alist = ["Alice", "Bob", "Carol"] - element, blist = peek(alist) + element, blist = peek(alist) element == alist[0] assert list(blist) == alist assert raises(StopIteration, lambda: peek([])) + + +def test_random_sample(): + alist = list(range(100)) + + assert list(random_sample(prob=1, seq=alist, random_state=2016)) == alist + + mk_rsample = lambda rs=1: list(random_sample(prob=0.1, + seq=alist, + random_state=rs)) + rsample1 = mk_rsample() + assert rsample1 == mk_rsample() + + rsample2 = mk_rsample(1984) + randobj = Random(1984) + assert rsample2 == mk_rsample(randobj) + + assert rsample1 != rsample2 + + assert mk_rsample(object) == mk_rsample(object) + assert mk_rsample(object) != mk_rsample(object()) + assert mk_rsample(b"a") == mk_rsample(u"a") + + assert raises(TypeError, lambda: mk_rsample([])) diff --git a/cytoolz/tests/test_none_safe.py b/cytoolz/tests/test_none_safe.py index 62f6280..cf6c7e8 100644 --- a/cytoolz/tests/test_none_safe.py +++ b/cytoolz/tests/test_none_safe.py @@ -100,6 +100,10 @@ def test_dicttoolz(): assert raises((AttributeError, TypeError), lambda: itemfilter(identity, None)) tested.append('itemfilter') + assert raises((AttributeError, TypeError), lambda: assoc_in(None, [2, 2], 3)) + assert raises(TypeError, lambda: assoc_in({}, None, 3)) + tested.append('assoc_in') + s1 = set(tested) s2 = set(cytoolz.dicttoolz.__all__) assert s1 == s2, '%s not tested for being None-safe' % ', '.join(s2 - s1) @@ -143,6 +147,10 @@ def test_functoolz(): assert flip(lambda a, b: (a, b))(None)(None) == (None, None) tested.append('flip') + excepts(None, lambda x: x) + excepts(TypeError, None) + tested.append('excepts') + s1 = set(tested) s2 = set(cytoolz.functoolz.__all__) assert s1 == s2, '%s not tested for being None-safe' % ', '.join(s2 - s1) @@ -294,6 +302,10 @@ def test_itertoolz(): assert raises(TypeError, lambda: peek(None)) tested.append('peek') + assert raises(TypeError, lambda: list(random_sample(None, [1]))) + assert raises(TypeError, lambda: list(random_sample(0.1, None))) + tested.append('random_sample') + s1 = set(tested) s2 = set(cytoolz.itertoolz.__all__) assert s1 == s2, '%s not tested for being None-safe' % ', '.join(s2 - s1) diff --git a/cytoolz/tests/test_signatures.py b/cytoolz/tests/test_signatures.py new file mode 100644 index 0000000..d395d08 --- /dev/null +++ b/cytoolz/tests/test_signatures.py @@ -0,0 +1,81 @@ +import functools +import sys +from cytoolz._signatures import (builtins, is_builtin_valid_args, + is_builtin_partial_args) +from cytoolz.compatibility import PY3 +from cytoolz.utils import raises + + +def test_is_valid(check_valid=is_builtin_valid_args, incomplete=False): + orig_check_valid = check_valid + check_valid = lambda func, *args, **kwargs: orig_check_valid(func, args, kwargs) + + assert check_valid(lambda x: None) is None + + f = builtins.abs + assert check_valid(f) is incomplete + assert check_valid(f, 1) + assert check_valid(f, x=1) is False + assert check_valid(f, 1, 2) is False + + f = builtins.complex + assert check_valid(f) + assert check_valid(f, 1) + assert check_valid(f, real=1) + assert check_valid(f, 1, 2) + assert check_valid(f, 1, imag=2) + assert check_valid(f, 1, real=2) is False + assert check_valid(f, 1, 2, 3) is False + assert check_valid(f, 1, 2, imag=3) is False + + f = builtins.int + assert check_valid(f) + assert check_valid(f, 1) + assert check_valid(f, x=1) + assert check_valid(f, 1, 2) + assert check_valid(f, 1, base=2) + assert check_valid(f, x=1, base=2) + assert check_valid(f, base=2) is incomplete + assert check_valid(f, 1, 2, 3) is False + + f = builtins.map + assert check_valid(f) is incomplete + assert check_valid(f, 1) is incomplete + assert check_valid(f, 1, 2) + assert check_valid(f, 1, 2, 3) + assert check_valid(f, 1, 2, 3, 4) + + f = builtins.min + assert check_valid(f) is incomplete + assert check_valid(f, 1) + assert check_valid(f, iterable=1) is False + assert check_valid(f, 1, 2) + assert check_valid(f, 1, 2, 3) + assert check_valid(f, key=None) is incomplete + assert check_valid(f, 1, key=None) + assert check_valid(f, 1, 2, key=None) + assert check_valid(f, 1, 2, 3, key=None) + assert check_valid(f, key=None, default=None) is (PY3 and incomplete) + assert check_valid(f, 1, key=None, default=None) is PY3 + assert check_valid(f, 1, 2, key=None, default=None) is False + assert check_valid(f, 1, 2, 3, key=None, default=None) is False + + f = builtins.range + assert check_valid(f) is incomplete + assert check_valid(f, 1) + assert check_valid(f, 1, 2) + assert check_valid(f, 1, 2, 3) + assert check_valid(f, 1, 2, step=3) is False + assert check_valid(f, 1, 2, 3, 4) is False + + f = functools.partial + assert orig_check_valid(f, (), {}) is incomplete + assert orig_check_valid(f, (), {'func': 1}) is incomplete + assert orig_check_valid(f, (1,), {}) + assert orig_check_valid(f, (1,), {'func': 1}) + assert orig_check_valid(f, (1, 2), {}) + + +def test_is_partial(): + test_is_valid(check_valid=is_builtin_partial_args, incomplete=True) + diff --git a/cytoolz/utils.pyx b/cytoolz/utils.pyx index 9c71b9d..20a20ef 100644 --- a/cytoolz/utils.pyx +++ b/cytoolz/utils.pyx @@ -1,4 +1,3 @@ -#cython: embedsignature=True import doctest import inspect import os.path diff --git a/setup.py b/setup.py index db8ddf6..5b31877 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,9 @@ ['cytoolz/' + modname + suffix])) if use_cython: + from Cython.Compiler.Options import directive_defaults + directive_defaults['embedsignature'] = True + directive_defaults['binding'] = True ext_modules = cythonize(ext_modules) setup( @@ -96,9 +99,9 @@ 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', 'Topic :: Scientific/Engineering', 'Topic :: Scientific/Engineering :: Information Analysis', 'Topic :: Software Development', From fe1ac2e21a4bb39a97cf7a54d376292c8b1a86f0 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 30 May 2016 13:28:48 -0500 Subject: [PATCH 092/146] Update introspection. `toolz` is now a dependency of `cytoolz`. Unfortunately, Cython builtins don't seem to provide a signature in Python 3, so we added virtually all of cytoolz to the signature registry. `cytoolz` now matches `toolz`. --- Makefile | 3 + cytoolz/__init__.py | 2 + cytoolz/_signatures.py | 904 +++++----------------------- cytoolz/compatibility.py | 1 + cytoolz/functoolz.pxd | 5 - cytoolz/functoolz.pyx | 239 ++------ cytoolz/tests/test_curried.py | 7 +- cytoolz/tests/test_functoolz.py | 16 +- cytoolz/tests/test_inspect_args.py | 261 +++++++- cytoolz/tests/test_serialization.py | 12 + cytoolz/tests/test_signatures.py | 18 +- 11 files changed, 477 insertions(+), 991 deletions(-) diff --git a/Makefile b/Makefile index 413fc46..efa19b3 100644 --- a/Makefile +++ b/Makefile @@ -5,3 +5,6 @@ inplace: test: inplace nosetests -s --with-doctest cytoolz/ + +clean: + rm cytoolz/*.c cytoolz/*.so diff --git a/cytoolz/__init__.py b/cytoolz/__init__.py index 15e003c..8c2eedd 100644 --- a/cytoolz/__init__.py +++ b/cytoolz/__init__.py @@ -17,4 +17,6 @@ # Aliases comp = compose +functoolz._sigs.update_signature_registry() + from ._version import __version__, __toolz_version__ diff --git a/cytoolz/_signatures.py b/cytoolz/_signatures.py index 340fdc6..0f84632 100644 --- a/cytoolz/_signatures.py +++ b/cytoolz/_signatures.py @@ -1,774 +1,154 @@ -"""Internal module for better introspection of builtins. - -The main functions are ``is_builtin_valid_args``, ``is_builtin_partial_args``, -and ``has_unknown_args``. Other functions in this module support these three. - -Notably, we create a ``signatures`` registry to enable introspection of -builtin functions in any Python version. This includes builtins that -have more than one valid signature. Currently, the registry includes -builtins from ``builtins``, ``functools``, ``itertools``, and ``operator`` -modules. More can be added as requested. We don't guarantee full coverage. - -Everything in this module should be regarded as implementation details. -Users should try to not use this module directly. - -""" -import functools -import inspect -import itertools -import operator -import sys - -from cytoolz.compatibility import PY3 - -if PY3: # pragma: py2 no cover - import builtins -else: # pragma: py3 no cover - import __builtin__ as builtins - -# We mock builtin callables using lists of tuples with lambda functions. -# -# The tuple spec is (num_position_args, lambda_func, keyword_only_args). -# -# num_position_args: -# - The number of positional-only arguments. If not specified, -# all positional arguments are considered positional-only. -# -# lambda_func: -# - lambda function that matches a signature of a builtin, but does -# not include keyword-only arguments. -# -# keyword_only_args: (optional) -# - Tuple of keyword-only argumemts. - -module_info = {} - -module_info[builtins] = dict( - abs=[ - lambda x: None], - all=[ - lambda iterable: None], - any=[ - lambda iterable: None], - apply=[ - lambda object: None, - lambda object, args: None, - lambda object, args, kwargs: None], - ascii=[ - lambda obj: None], - bin=[ - lambda number: None], - bool=[ - lambda x=False: None], - buffer=[ - lambda object: None, - lambda object, offset: None, - lambda object, offset, size: None], - bytearray=[ - lambda: None, - lambda int: None, - lambda string, encoding='utf8', errors='strict': None], - callable=[ - lambda obj: None], - chr=[ - lambda i: None], - classmethod=[ - lambda function: None], - cmp=[ - lambda x, y: None], - coerce=[ - lambda x, y: None], - complex=[ - lambda real=0, imag=0: None], - delattr=[ - lambda obj, name: None], - dict=[ - lambda **kwargs: None, - lambda mapping, **kwargs: None], - dir=[ - lambda: None, - lambda object: None], - divmod=[ - lambda x, y: None], - enumerate=[ - (0, lambda iterable, start=0: None)], - eval=[ - lambda source: None, - lambda source, globals: None, - lambda source, globals, locals: None], - execfile=[ - lambda filename: None, - lambda filename, globals: None, - lambda filename, globals, locals: None], - file=[ - (0, lambda name, mode='r', buffering=-1: None)], - filter=[ - lambda function, iterable: None], - float=[ - lambda x=0.0: None], - format=[ - lambda value: None, - lambda value, format_spec: None], - frozenset=[ - lambda: None, - lambda iterable: None], - getattr=[ - lambda object, name: None, - lambda object, name, default: None], - globals=[ - lambda: None], - hasattr=[ - lambda obj, name: None], - hash=[ - lambda obj: None], - hex=[ - lambda number: None], - id=[ - lambda obj: None], - input=[ - lambda: None, - lambda prompt: None], - int=[ - lambda x=0: None, - (0, lambda x, base=10: None)], - intern=[ - lambda string: None], - isinstance=[ - lambda obj, class_or_tuple: None], - issubclass=[ - lambda cls, class_or_tuple: None], - iter=[ - lambda iterable: None, - lambda callable, sentinel: None], - len=[ - lambda obj: None], - list=[ - lambda: None, - lambda iterable: None], - locals=[ - lambda: None], - long=[ - lambda x=0: None, - (0, lambda x, base=10: None)], - map=[ - lambda func, sequence, *iterables: None], - memoryview=[ - (0, lambda object: None)], - next=[ - lambda iterator: None, - lambda iterator, default: None], - object=[ - lambda: None], - oct=[ - lambda number: None], - ord=[ - lambda c: None], - pow=[ - lambda x, y: None, - lambda x, y, z: None], - property=[ - lambda fget=None, fset=None, fdel=None, doc=None: None], - range=[ - lambda stop: None, - lambda start, stop: None, - lambda start, stop, step: None], - raw_input=[ - lambda: None, - lambda prompt: None], - reduce=[ - lambda function, sequence: None, - lambda function, sequence, initial: None], - reload=[ - lambda module: None], - repr=[ - lambda obj: None], - reversed=[ - lambda sequence: None], - round=[ - (0, lambda number, ndigits=0: None)], - set=[ - lambda: None, - lambda iterable: None], - setattr=[ - lambda obj, name, value: None], - slice=[ - lambda stop: None, - lambda start, stop: None, - lambda start, stop, step: None], - staticmethod=[ - lambda function: None], - sum=[ - lambda iterable: None, - lambda iterable, start: None], - super=[ - lambda type: None, - lambda type, obj: None], - tuple=[ - lambda: None, - lambda iterable: None], - type=[ - lambda object: None, - lambda name, bases, dict: None], - unichr=[ - lambda i: None], - unicode=[ - lambda object: None, - lambda string='', encoding='utf8', errors='strict': None], - vars=[ - lambda: None, - lambda object: None], - xrange=[ - lambda stop: None, - lambda start, stop: None, - lambda start, stop, step: None], - zip=[ - lambda *iterables: None], +from toolz._signatures import * +from toolz._signatures import (_is_arity, _has_varargs, _has_keywords, + _num_required_args, _is_partial_args, _is_valid_args) + +cytoolz_info = {} + +cytoolz_info['cytoolz.dicttoolz'] = dict( + assoc=[ + lambda d, key, value, factory=dict: None], + assoc_in=[ + lambda d, keys, value, factory=dict: None], + dissoc=[ + lambda d, *keys: None], + get_in=[ + lambda keys, coll, default=None, no_default=False: None], + itemfilter=[ + lambda predicate, d, factory=dict: None], + itemmap=[ + lambda func, d, factory=dict: None], + keyfilter=[ + lambda predicate, d, factory=dict: None], + keymap=[ + lambda func, d, factory=dict: None], + merge=[ + lambda *dicts, **kwargs: None], + merge_with=[ + lambda func, *dicts, **kwargs: None], + update_in=[ + lambda d, keys, func, default=None, factory=dict: None], + valfilter=[ + lambda predicate, d, factory=dict: None], + valmap=[ + lambda func, d, factory=dict: None], ) -module_info[builtins]['exec'] = [ - lambda source: None, - lambda source, globals: None, - lambda source, globals, locals: None] - -if PY3: # pragma: py2 no cover - module_info[builtins].update( - bytes=[ - lambda: None, - lambda int: None, - lambda string, encoding='utf8', errors='strict': None], - compile=[ - (0, lambda source, filename, mode, flags=0, - dont_inherit=False, optimize=-1: None)], - max=[ - (1, lambda iterable: None, ('default', 'key',)), - (1, lambda arg1, arg2, *args: None, ('key',))], - min=[ - (1, lambda iterable: None, ('default', 'key',)), - (1, lambda arg1, arg2, *args: None, ('key',))], - open=[ - (0, lambda file, mode='r', buffering=-1, encoding=None, - errors=None, newline=None, closefd=True, opener=None: None)], - sorted=[ - (1, lambda iterable: None, ('key', 'reverse'))], - str=[ - lambda object='', encoding='utf', errors='strict': None], - ) - module_info[builtins]['print'] = [ - (0, lambda *args: None, ('sep', 'end', 'file', 'flush',))] - -else: # pragma: py3 no cover - module_info[builtins].update( - bytes=[ - lambda object='': None], - compile=[ - (0, lambda source, filename, mode, flags=0, - dont_inherit=False: None)], - max=[ - (1, lambda iterable, *args: None, ('key',))], - min=[ - (1, lambda iterable, *args: None, ('key',))], - open=[ - (0, lambda file, mode='r', buffering=-1: None)], - sorted=[ - lambda iterable, cmp=None, key=None, reverse=False: None], - str=[ - lambda object='': None], - ) - module_info[builtins]['print'] = [ - (0, lambda *args: None, ('sep', 'end', 'file',))] -module_info[functools] = dict( - cmp_to_key=[ - (0, lambda mycmp: None)], - partial=[ - lambda func, *args, **kwargs: None], - partialmethod=[ - lambda func, *args, **kwargs: None], - reduce=[ - lambda function, sequence: None, - lambda function, sequence, initial: None], +cytoolz_info['cytoolz.functoolz'] = dict( + Compose=[ + lambda *funcs: None], + _flip=[ + lambda f, a, b: None], + c_memoize=[ + lambda func, cache=None, key=None: None], + complement=[ + lambda func: None], + compose=[ + lambda *funcs: None], + curry=[ + lambda *args, **kwargs: None], + do=[ + lambda func, x: None], + excepts=[ + lambda exc, func, handler=None: None], + identity=[ + lambda x: None], + juxt=[ + lambda *funcs: None], + memoize=[ + lambda func=None, cache=None, key=None: None], + pipe=[ + lambda data, *funcs: None], + return_none=[ + lambda exc: None], + thread_first=[ + lambda val, *forms: None], + thread_last=[ + lambda val, *forms: None], ) -module_info[itertools] = dict( +cytoolz_info['cytoolz.itertoolz'] = dict( accumulate=[ - (0, lambda iterable, func=None: None)], - chain=[ - lambda *iterables: None], - combinations=[ - (0, lambda iterable, r: None)], - combinations_with_replacement=[ - (0, lambda iterable, r: None)], - compress=[ - (0, lambda data, selectors: None)], + lambda binop, seq, initial='__no__default__': None], + cons=[ + lambda el, seq: None], count=[ - lambda start=0, step=1: None], - cycle=[ - lambda iterable: None], - dropwhile=[ - lambda predicate, iterable: None], - filterfalse=[ - lambda function, sequence: None], + lambda seq: None], + diff=[ + lambda *seqs, **kwargs: None], + drop=[ + lambda n, seq: None], + first=[ + lambda seq: None], + frequencies=[ + lambda seq: None], + get=[ + lambda ind, seq, default=None: None], + getter=[ + lambda index: None], groupby=[ - (0, lambda iterable, key=None: None)], - ifilter=[ - lambda function, sequence: None], - ifilterfalse=[ - lambda function, sequence: None], - imap=[ - lambda func, sequence, *iterables: None], - islice=[ - lambda iterable, stop: None, - lambda iterable, start, stop: None, - lambda iterable, start, stop, step: None], - izip=[ - lambda *iterables: None], - izip_longest=[ - (0, lambda *iterables: None, ('fillvalue',))], - permutations=[ - (0, lambda iterable, r=0: None)], - repeat=[ - (0, lambda object, times=0: None)], - starmap=[ - lambda function, sequence: None], - takewhile=[ - lambda predicate, iterable: None], - tee=[ - lambda iterable: None, - lambda iterable, n: None], - zip_longest=[ - (0, lambda *iterables: None, ('fillvalue',))], -) - -if PY3: # pragma: py2 no cover - module_info[itertools].update( - product=[ - (0, lambda *iterables: None, ('repeat',))], - ) -else: # pragma: py3 no cover - module_info[itertools].update( - product=[ - lambda *iterables: None], - ) - -module_info[operator] = dict( - __abs__=[ - lambda a: None], - __add__=[ - lambda a, b: None], - __and__=[ - lambda a, b: None], - __concat__=[ - lambda a, b: None], - __contains__=[ - lambda a, b: None], - __delitem__=[ - lambda a, b: None], - __delslice__=[ - lambda a, b, c: None], - __div__=[ - lambda a, b: None], - __eq__=[ - lambda a, b: None], - __floordiv__=[ - lambda a, b: None], - __ge__=[ - lambda a, b: None], - __getitem__=[ - lambda a, b: None], - __getslice__=[ - lambda a, b, c: None], - __gt__=[ - lambda a, b: None], - __iadd__=[ - lambda a, b: None], - __iand__=[ - lambda a, b: None], - __iconcat__=[ - lambda a, b: None], - __idiv__=[ - lambda a, b: None], - __ifloordiv__=[ - lambda a, b: None], - __ilshift__=[ - lambda a, b: None], - __imatmul__=[ - lambda a, b: None], - __imod__=[ - lambda a, b: None], - __imul__=[ - lambda a, b: None], - __index__=[ - lambda a: None], - __inv__=[ - lambda a: None], - __invert__=[ - lambda a: None], - __ior__=[ - lambda a, b: None], - __ipow__=[ - lambda a, b: None], - __irepeat__=[ - lambda a, b: None], - __irshift__=[ - lambda a, b: None], - __isub__=[ - lambda a, b: None], - __itruediv__=[ - lambda a, b: None], - __ixor__=[ - lambda a, b: None], - __le__=[ - lambda a, b: None], - __lshift__=[ - lambda a, b: None], - __lt__=[ - lambda a, b: None], - __matmul__=[ - lambda a, b: None], - __mod__=[ - lambda a, b: None], - __mul__=[ - lambda a, b: None], - __ne__=[ - lambda a, b: None], - __neg__=[ - lambda a: None], - __not__=[ - lambda a: None], - __or__=[ - lambda a, b: None], - __pos__=[ - lambda a: None], - __pow__=[ - lambda a, b: None], - __repeat__=[ - lambda a, b: None], - __rshift__=[ - lambda a, b: None], - __setitem__=[ - lambda a, b, c: None], - __setslice__=[ - lambda a, b, c, d: None], - __sub__=[ - lambda a, b: None], - __truediv__=[ - lambda a, b: None], - __xor__=[ - lambda a, b: None], - _abs=[ + lambda key, seq: None], + identity=[ + lambda x: None], + interleave=[ + lambda seqs, pass_exceptions=(): None], + interpose=[ + lambda el, seq: None], + isdistinct=[ + lambda seq: None], + isiterable=[ lambda x: None], - _compare_digest=[ - lambda a, b: None], - abs=[ - lambda a: None], - add=[ - lambda a, b: None], - and_=[ - lambda a, b: None], - attrgetter=[ - lambda attr, *args: None], - concat=[ - lambda a, b: None], - contains=[ - lambda a, b: None], - countOf=[ - lambda a, b: None], - delitem=[ - lambda a, b: None], - delslice=[ - lambda a, b, c: None], - div=[ - lambda a, b: None], - eq=[ - lambda a, b: None], - floordiv=[ - lambda a, b: None], - ge=[ - lambda a, b: None], - getitem=[ - lambda a, b: None], - getslice=[ - lambda a, b, c: None], - gt=[ - lambda a, b: None], - iadd=[ - lambda a, b: None], - iand=[ - lambda a, b: None], - iconcat=[ - lambda a, b: None], - idiv=[ - lambda a, b: None], - ifloordiv=[ - lambda a, b: None], - ilshift=[ - lambda a, b: None], - imatmul=[ - lambda a, b: None], - imod=[ - lambda a, b: None], - imul=[ - lambda a, b: None], - index=[ - lambda a: None], - indexOf=[ - lambda a, b: None], - inv=[ - lambda a: None], - invert=[ - lambda a: None], - ior=[ - lambda a, b: None], - ipow=[ - lambda a, b: None], - irepeat=[ - lambda a, b: None], - irshift=[ - lambda a, b: None], - is_=[ - lambda a, b: None], - is_not=[ - lambda a, b: None], - isCallable=[ - lambda a: None], - isMappingType=[ - lambda a: None], - isNumberType=[ - lambda a: None], - isSequenceType=[ - lambda a: None], - isub=[ - lambda a, b: None], - itemgetter=[ - lambda item, *args: None], - itruediv=[ - lambda a, b: None], - ixor=[ - lambda a, b: None], - le=[ - lambda a, b: None], - length_hint=[ - lambda obj: None, - lambda obj, default: None], - lshift=[ - lambda a, b: None], - lt=[ - lambda a, b: None], - matmul=[ - lambda a, b: None], - methodcaller=[ - lambda name, *args, **kwargs: None], - mod=[ - lambda a, b: None], - mul=[ - lambda a, b: None], - ne=[ - lambda a, b: None], - neg=[ - lambda a: None], - not_=[ - lambda a: None], - or_=[ - lambda a, b: None], - pos=[ - lambda a: None], - pow=[ - lambda a, b: None], - repeat=[ - lambda a, b: None], - rshift=[ - lambda a, b: None], - sequenceIncludes=[ - lambda a, b: None], - setitem=[ - lambda a, b, c: None], - setslice=[ - lambda a, b, c, d: None], - sub=[ - lambda a, b: None], - truediv=[ - lambda a, b: None], - truth=[ - lambda a: None], - xor=[ - lambda a, b: None], + iterate=[ + lambda func, x: None], + join=[ + lambda leftkey, leftseq, rightkey, rightseq, left_default=None, right_default=None: None], + last=[ + lambda seq: None], + mapcat=[ + lambda func, seqs: None], + merge_sorted=[ + lambda *seqs, **kwargs: None], + nth=[ + lambda n, seq: None], + partition=[ + lambda n, seq, pad=None: None], + partition_all=[ + lambda n, seq: None], + peek=[ + lambda seq: None], + pluck=[ + lambda ind, seqs, default=None: None], + random_sample=[ + lambda prob, seq, random_state=None: None], + reduceby=[ + lambda key, binop, seq, init=None: None], + remove=[ + lambda predicate, seq: None], + rest=[ + lambda seq: None], + second=[ + lambda seq: None], + sliding_window=[ + lambda n, seq: None], + tail=[ + lambda n, seq: None], + take=[ + lambda n, seq: None], + take_nth=[ + lambda n, seq: None], + topk=[ + lambda k, seq, key=None: None], + unique=[ + lambda seq, key=None: None], ) -if PY3: # pragma: py2 no cover - def num_pos_args(func, sigspec): - """Return the number of positional arguments. ``f(x, y=1)`` has 1.""" - return sum(1 for x in sigspec.parameters.values() - if x.kind == x.POSITIONAL_OR_KEYWORD and - x.default is x.empty) - - def get_exclude_keywords(func, num_pos_only, sigspec): - """Return the names of position-only arguments if func has **kwargs""" - if num_pos_only == 0: - return () - has_kwargs = any(x.kind == x.VAR_KEYWORD - for x in sigspec.parameters.values()) - if not has_kwargs: - return () - pos_args = list(sigspec.parameters.values())[:num_pos_only] - return tuple(x.name for x in pos_args) - - def signature_or_spec(func): - try: - return inspect.signature(func) - except (ValueError, TypeError) as e: - return e - -else: # pragma: py3 no cover - def num_pos_args(func, sigspec): - """Return the number of positional arguments. ``f(x, y=1)`` has 1.""" - if sigspec.defaults: - return len(sigspec.args) - len(sigspec.defaults) - return len(sigspec.args) - - def get_exclude_keywords(func, num_pos_only, sigspec): - """Return the names of position-only arguments if func has **kwargs""" - if num_pos_only == 0: - return () - has_kwargs = sigspec.keywords is not None - if not has_kwargs: - return () - return tuple(sigspec.args[:num_pos_only]) - - def signature_or_spec(func): - try: - return inspect.getargspec(func) - except TypeError as e: - return e - - -def expand_sig(sig): - """Convert the signature spec in ``module_info`` to add to ``signatures``. - - The input signature spec is one of: - - ``lambda_func`` - - ``(num_position_args, lambda_func)`` - - ``(num_position_args, lambda_func, keyword_only_args)`` - - The output signature spec is: - ``(num_position_args, lambda_func, keyword_exclude, sigspec)`` - - where ``keyword_exclude`` includes keyword only arguments and, if variadic - keywords is present, the names of position-only argument. The latter is - included to support builtins such as ``partial(func, *args, **kwargs)``, - which allows ``func=`` to be used as a keyword even though it's the name - of a positional argument. - - """ - if isinstance(sig, tuple): - if len(sig) == 3: - num_pos_only, func, keyword_only = sig - assert isinstance(sig[-1], tuple) - else: - num_pos_only, func = sig - keyword_only = () - sigspec = signature_or_spec(func) - else: - func = sig - sigspec = signature_or_spec(func) - num_pos_only = num_pos_args(func, sigspec) - keyword_only = () - keyword_exclude = get_exclude_keywords(func, num_pos_only, sigspec) - return (num_pos_only, func, keyword_only + keyword_exclude, sigspec) - - -signatures = {} -for module, info in module_info.items(): - for name, sigs in info.items(): - if hasattr(module, name): - new_sigs = tuple(expand_sig(sig) for sig in sigs) - signatures[getattr(module, name)] = new_sigs - - -def check_valid(sig, args, kwargs): - """Like ``is_valid_args`` for the given signature spec.""" - num_pos_only, func, keyword_exclude, sigspec = sig - if len(args) < num_pos_only: - return False - if keyword_exclude: - kwargs = dict(kwargs) - for item in keyword_exclude: - kwargs.pop(item, None) - try: - func(*args, **kwargs) - return True - except TypeError: - return False - - -def check_partial(sig, args, kwargs): - """Like ``is_partial_args`` for the given signature spec.""" - num_pos_only, func, keyword_exclude, sigspec = sig - if len(args) < num_pos_only: - pad = (None,) * (num_pos_only - len(args)) - args = args + pad - if keyword_exclude: - kwargs = dict(kwargs) - for item in keyword_exclude: - kwargs.pop(item, None) - return is_partial_args(func, args, kwargs, sigspec=sigspec) - - -def is_builtin_valid_args(func, args, kwargs): - """Like ``is_valid_args`` for builtins in our ``signatures`` registry.""" - if func not in signatures: - return None - sigs = signatures[func] - return any(check_valid(sig, args, kwargs) for sig in sigs) - - -def is_builtin_partial_args(func, args, kwargs): - """Like ``is_partial_args`` for builtins in our ``signatures`` registry.""" - if func not in signatures: - return None - sigs = signatures[func] - return any(check_partial(sig, args, kwargs) for sig in sigs) - - -if PY3: # pragma: py2 no cover - def has_unknown_args(func, sigspec=None): - """Might ``func`` have ``*args`` that is passed to a wrapped function? - - This is specifically to support ``curry``. - - """ - if func in signatures: - return False - if sigspec is None: - try: - sigspec = inspect.signature(func) - except (ValueError, TypeError) as e: - sigspec = e - if isinstance(sigspec, ValueError): - return True - elif isinstance(sigspec, TypeError): - return False - try: - return any(x.kind == x.VAR_POSITIONAL - for x in sigspec.parameters.values()) - except AttributeError: # pragma: no cover - return False - -else: # pragma: py3 no cover - def has_unknown_args(func, sigspec=None): - """Might ``func`` have ``*args`` that is passed to a wrapped function? - - This is specifically to support ``curry``. - - """ - if func in signatures: - return False - if sigspec is None: - try: - sigspec = inspect.getargspec(func) - except TypeError as e: - sigspec = e - if isinstance(sigspec, TypeError): - return callable(func) - return sigspec.varargs is not None +cytoolz_info['cytoolz.recipes'] = dict( + countby=[ + lambda key, seq: None], + partitionby=[ + lambda func, seq: None], +) +def update_signature_registry(): + create_signature_registry(cytoolz_info) + module_info.update(cytoolz_info) -from .functoolz import is_partial_args diff --git a/cytoolz/compatibility.py b/cytoolz/compatibility.py index ad34a83..fef5acc 100644 --- a/cytoolz/compatibility.py +++ b/cytoolz/compatibility.py @@ -1,6 +1,7 @@ import operator import sys PY3 = sys.version_info[0] > 2 +PY33 = sys.version_info[0] == 3 and sys.version_info[1] == 3 PY34 = sys.version_info[0] == 3 and sys.version_info[1] == 4 __all__ = ['PY3', 'map', 'filter', 'range', 'zip', 'reduce', 'zip_longest', diff --git a/cytoolz/functoolz.pxd b/cytoolz/functoolz.pxd index 0c8e871..5e6d2f4 100644 --- a/cytoolz/functoolz.pxd +++ b/cytoolz/functoolz.pxd @@ -60,8 +60,3 @@ cdef class excepts: cdef public object func cdef public object handler - -cpdef object is_valid_args(object func, object args, object kwargs, object sigspec=*) - - -cpdef object is_partial_args(object func, object args, object kwargs, object sigspec=*) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 6b3ecfa..bf9433b 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -3,13 +3,14 @@ import sys from functools import partial from operator import attrgetter from textwrap import dedent +from cytoolz.utils import no_default from cytoolz.compatibility import PY3, PY34, filter as ifilter, map as imap, reduce -from cytoolz._signatures import ( - is_builtin_valid_args as _is_builtin_valid_args, - is_builtin_partial_args as _is_builtin_partial_args, - has_unknown_args as _has_unknown_args, - signature_or_spec as _signature_or_spec, -) +import cytoolz._signatures as _sigs + +from toolz.functoolz import (InstanceProperty, instanceproperty, is_arity, + num_required_args, has_varargs, has_keywords, + is_valid_args, is_partial_args) + from cpython.dict cimport PyDict_Merge, PyDict_New from cpython.exc cimport PyErr_Clear, PyErr_Occurred, PyErr_GivenExceptionMatches from cpython.object cimport (PyCallable_Check, PyObject_Call, PyObject_CallObject, @@ -169,9 +170,6 @@ cdef class curry: cytoolz.curried - namespace of curried functions http://toolz.readthedocs.org/en/latest/curry.html """ - property __wrapped__: - def __get__(self): - return self.func def __cinit__(self, *args, **kwargs): if not args: @@ -256,12 +254,12 @@ cdef class curry: # kwargs = dict(self.keywords, **kwargs) if self._sigspec is None: - sigspec = self._sigspec = _signature_or_spec(func) - self._has_unknown_args = _has_unknown_args(func, sigspec=sigspec) + sigspec = self._sigspec = _sigs.signature_or_spec(func) + self._has_unknown_args = has_varargs(func, sigspec) is not False else: sigspec = self._sigspec - if is_partial_args(func, args, kwargs, sigspec=sigspec) is False: + if is_partial_args(func, args, kwargs, sigspec) is False: # Nothing can make the call valid return False elif self._has_unknown_args: @@ -269,7 +267,7 @@ cdef class curry: # anyway because the function may have `*args`. This is useful # for decorators with signature `func(*args, **kwargs)`. return True - elif not is_valid_args(func, args, kwargs, sigspec=sigspec): + elif not is_valid_args(func, args, kwargs, sigspec): # Adding more arguments may make the call valid return True else: @@ -311,48 +309,43 @@ cdef class curry: def __setstate__(self, state): self.args, self.keywords = state + property __signature__: + def __get__(self): + sig = inspect.signature(self.func) + args = self.args or () + keywords = self.keywords or {} + if is_partial_args(self.func, args, keywords, sig) is False: + raise TypeError('curry object has incorrect arguments') + + params = list(sig.parameters.values()) + skip = 0 + for param in params[:len(args)]: + if param.kind == param.VAR_POSITIONAL: + break + skip += 1 + + kwonly = False + newparams = [] + for param in params[skip:]: + kind = param.kind + default = param.default + if kind == param.VAR_KEYWORD: + pass + elif kind == param.VAR_POSITIONAL: + if kwonly: + continue + elif param.name in keywords: + default = keywords[param.name] + kind = param.KEYWORD_ONLY + kwonly = True + else: + if kwonly: + kind = param.KEYWORD_ONLY + if default is param.empty: + default = no_default + newparams.append(param.replace(default=default, kind=kind)) -cpdef object has_kwargs(object f): - """ - Does a function have keyword arguments? - - >>> def f(x, y=0): - ... return x + y - - >>> has_kwargs(f) - True - """ - if sys.version_info[0] == 2: - spec = inspect.getargspec(f) - return bool(spec and (spec.keywords or spec.defaults)) - if sys.version_info[0] == 3: - spec = inspect.getfullargspec(f) - return bool(spec.defaults) - - -cpdef object isunary(object f): - """ - Does a function have only a single argument? - - >>> def f(x): - ... return x - - >>> isunary(f) - True - >>> isunary(lambda x, y: x + y) - False - """ - cdef int major = sys.version_info[0] - try: - if major == 2: - spec = inspect.getargspec(f) - if major == 3: - spec = inspect.getfullargspec(f) - return bool(spec and spec.varargs is None and not has_kwargs(f) - and len(spec.args) == 1) - except TypeError: - pass - return None # in Python < 3.4 builtins fail, return None + return sig.replace(parameters=newparams) cdef class c_memoize: @@ -377,9 +370,9 @@ cdef class c_memoize: self.key = key try: - self.may_have_kwargs = has_kwargs(func) + self.may_have_kwargs = has_keywords(func) is not False # Is unary function (single arg, no variadic argument or keywords)? - self.is_unary = isunary(func) + self.is_unary = is_arity(1, func) except TypeError: self.is_unary = False self.may_have_kwargs = True @@ -757,137 +750,3 @@ cdef class excepts: except AttributeError: return type(self).__doc__ - -cpdef object is_valid_args(object func, object args, object kwargs, object sigspec=None): - if PY34: - val = _is_builtin_valid_args(func, args, kwargs) - if val is not None: - return val - if PY3: - if sigspec is None: - try: - sigspec = inspect.signature(func) - except (ValueError, TypeError) as e: - sigspec = e - if isinstance(sigspec, ValueError): - return _is_builtin_valid_args(func, args, kwargs) - elif isinstance(sigspec, TypeError): - return False - try: - sigspec.bind(*args, **kwargs) - except (TypeError, AttributeError): - return False - return True - - else: - if sigspec is None: - try: - sigspec = inspect.getargspec(func) - except TypeError as e: - sigspec = e - if isinstance(sigspec, TypeError): - if not callable(func): - return False - return _is_builtin_valid_args(func, args, kwargs) - - spec = sigspec - defaults = spec.defaults or () - num_pos = len(spec.args) - len(defaults) - missing_pos = spec.args[len(args):num_pos] - for arg in missing_pos: - if arg not in kwargs: - return False - - if spec.varargs is None: - num_extra_pos = max(0, len(args) - num_pos) - else: - num_extra_pos = 0 - - kwargs = dict(kwargs) - - # Add missing keyword arguments (unless already included in `args`) - missing_kwargs = spec.args[num_pos + num_extra_pos:] - kwargs.update(zip(missing_kwargs, defaults[num_extra_pos:])) - - # Convert call to use positional arguments - more_args = [] - for key in spec.args[len(args):]: - more_args.append(kwargs.pop(key)) - args = args + tuple(more_args) - - if ( - not spec.keywords and kwargs or - not spec.varargs and len(args) > len(spec.args) or - set(spec.args[:len(args)]) & set(kwargs) - ): - return False - else: - return True - - -cpdef object is_partial_args(object func, object args, object kwargs, object sigspec=None): - if PY34: - val = _is_builtin_partial_args(func, args, kwargs) - if val is not None: - return val - if PY3: - if sigspec is None: - try: - sigspec = inspect.signature(func) - except (ValueError, TypeError) as e: - sigspec = e - if isinstance(sigspec, ValueError): - return _is_builtin_partial_args(func, args, kwargs) - elif isinstance(sigspec, TypeError): - return False - try: - sigspec.bind_partial(*args, **kwargs) - except (TypeError, AttributeError): - return False - return True - - else: - if sigspec is None: - try: - sigspec = inspect.getargspec(func) - except TypeError as e: - sigspec = e - if isinstance(sigspec, TypeError): - if not callable(func): - return False - return _is_builtin_partial_args(func, args, kwargs) - - spec = sigspec - defaults = spec.defaults or () - num_pos = len(spec.args) - len(defaults) - if spec.varargs is None: - num_extra_pos = max(0, len(args) - num_pos) - else: - num_extra_pos = 0 - - kwargs = dict(kwargs) - - # Add missing keyword arguments (unless already included in `args`) - missing_kwargs = spec.args[num_pos + num_extra_pos:] - kwargs.update(zip(missing_kwargs, defaults[num_extra_pos:])) - - # Add missing position arguments as keywords (may already be in kwargs) - missing_args = spec.args[len(args):num_pos + num_extra_pos] - for x in missing_args: - kwargs[x] = None - - # Convert call to use positional arguments - more_args = [] - for key in spec.args[len(args):]: - more_args.append(kwargs.pop(key)) - args = args + tuple(more_args) - - if ( - not spec.keywords and kwargs or - not spec.varargs and len(args) > len(spec.args) or - set(spec.args[:len(args)]) & set(kwargs) - ): - return False - else: - return True - diff --git a/cytoolz/tests/test_curried.py b/cytoolz/tests/test_curried.py index 0665d46..da15ff8 100644 --- a/cytoolz/tests/test_curried.py +++ b/cytoolz/tests/test_curried.py @@ -48,9 +48,14 @@ def test_curried_operator(): if not isinstance(v, cytoolz.curry): try: # Make sure it is unary - # We cannot use isunary because it might be defined in C. v(1) except TypeError: + try: + v('x') + except TypeError: + pass + else: + continue raise AssertionError( 'cytoolz.curried.operator.%s is not curried!' % k, ) diff --git a/cytoolz/tests/test_functoolz.py b/cytoolz/tests/test_functoolz.py index 087cd8b..5aa0723 100644 --- a/cytoolz/tests/test_functoolz.py +++ b/cytoolz/tests/test_functoolz.py @@ -5,7 +5,7 @@ from operator import add, mul, itemgetter from cytoolz.utils import raises from functools import partial -from cytoolz.compatibility import PY3 + def iseven(x): return x % 2 == 0 @@ -277,6 +277,9 @@ def foo(a, b, c=1): assert raises(AttributeError, lambda: setattr(f, 'args', (2,))) assert raises(AttributeError, lambda: setattr(f, 'keywords', {'c': 3})) assert raises(AttributeError, lambda: setattr(f, 'func', f)) + assert raises(AttributeError, lambda: delattr(f, 'args')) + assert raises(AttributeError, lambda: delattr(f, 'keywords')) + assert raises(AttributeError, lambda: delattr(f, 'func')) def test_curry_attributes_writable(): @@ -422,17 +425,6 @@ def __hash__(self): assert A.addstatic(3, 4) == 7 -def test_curry_wrapped(): - - def foo(a): - """ - Docstring - """ - pass - curried_foo = curry(foo) - assert curried_foo.__wrapped__ is foo - - def test_curry_call(): @curry def add(x, y): diff --git a/cytoolz/tests/test_inspect_args.py b/cytoolz/tests/test_inspect_args.py index 881d478..061810f 100644 --- a/cytoolz/tests/test_inspect_args.py +++ b/cytoolz/tests/test_inspect_args.py @@ -1,8 +1,13 @@ import functools -import sys -from cytoolz.functoolz import curry, is_valid_args, is_partial_args -from cytoolz._signatures import has_unknown_args -from cytoolz.compatibility import PY3 +import inspect +import itertools +import operator +import cytoolz +from cytoolz.functoolz import (curry, is_valid_args, is_partial_args, is_arity, + num_required_args, has_varargs, has_keywords) +from cytoolz._signatures import builtins +import cytoolz._signatures as _sigs +from cytoolz.compatibility import PY3, PY33 from cytoolz.utils import raises @@ -240,20 +245,20 @@ def f(func=None): def test_has_unknown_args(): - assert has_unknown_args(1) is False - assert has_unknown_args(map) is False - assert has_unknown_args(make_func('')) is False - assert has_unknown_args(make_func('x, y, z')) is False - assert has_unknown_args(make_func('*args')) - assert has_unknown_args(make_func('**kwargs')) is False - assert has_unknown_args(make_func('x, y, *args, **kwargs')) - assert has_unknown_args(make_func('x, y, z=1')) is False - assert has_unknown_args(make_func('x, y, z=1, **kwargs')) is False + assert has_varargs(1) is False + assert has_varargs(map) + assert has_varargs(make_func('')) is False + assert has_varargs(make_func('x, y, z')) is False + assert has_varargs(make_func('*args')) + assert has_varargs(make_func('**kwargs')) is False + assert has_varargs(make_func('x, y, *args, **kwargs')) + assert has_varargs(make_func('x, y, z=1')) is False + assert has_varargs(make_func('x, y, z=1, **kwargs')) is False if PY3: f = make_func('*args') f.__signature__ = 34 - assert has_unknown_args(f) is False + assert has_varargs(f) is False class RaisesValueError(object): def __call__(self): @@ -263,5 +268,231 @@ def __signature__(self): raise ValueError('Testing Python 3.4') f = RaisesValueError() - assert has_unknown_args(f) + assert has_varargs(f) is None + + +def test_num_required_args(): + assert num_required_args(lambda: None) == 0 + assert num_required_args(lambda x: None) == 1 + assert num_required_args(lambda x, *args: None) == 1 + assert num_required_args(lambda x, **kwargs: None) == 1 + assert num_required_args(lambda x, y, *args, **kwargs: None) == 2 + assert num_required_args(map) == 2 + assert num_required_args(dict) is None + + +def test_has_keywords(): + assert has_keywords(lambda: None) is False + assert has_keywords(lambda x: None) is False + assert has_keywords(lambda x=1: None) + assert has_keywords(lambda **kwargs: None) + assert has_keywords(int) + assert has_keywords(sorted) + assert has_keywords(max) + assert has_keywords(map) is False + assert has_keywords(bytearray) is None + + +def test_has_varargs(): + assert has_varargs(lambda: None) is False + assert has_varargs(lambda *args: None) + assert has_varargs(lambda **kwargs: None) is False + assert has_varargs(map) + if PY3: + assert has_varargs(max) is None + + +def test_is_arity(): + assert is_arity(0, lambda: None) + assert is_arity(1, lambda: None) is False + assert is_arity(1, lambda x: None) + assert is_arity(3, lambda x, y, z: None) + assert is_arity(1, lambda x, *args: None) is False + assert is_arity(1, lambda x, **kwargs: None) is False + assert is_arity(1, all) + assert is_arity(2, map) is False + assert is_arity(2, range) is None + + +def test_introspect_curry_valid_py3(check_valid=is_valid_args, incomplete=False): + if not PY3: + return + orig_check_valid = check_valid + check_valid = lambda _func, *args, **kwargs: orig_check_valid(_func, args, kwargs) + + f = cytoolz.curry(make_func('x, y, z=0')) + assert check_valid(f) + assert check_valid(f, 1) + assert check_valid(f, 1, 2) + assert check_valid(f, 1, 2, 3) + assert check_valid(f, 1, 2, 3, 4) is False + assert check_valid(f, invalid_keyword=True) is False + assert check_valid(f(1)) + assert check_valid(f(1), 2) + assert check_valid(f(1), 2, 3) + assert check_valid(f(1), 2, 3, 4) is False + assert check_valid(f(1), x=2) is False + assert check_valid(f(1), y=2) + assert check_valid(f(x=1), 2) is False + assert check_valid(f(x=1), y=2) + assert check_valid(f(y=2), 1) + assert check_valid(f(y=2), 1, z=3) + assert check_valid(f(y=2), 1, 3) is False + + f = cytoolz.curry(make_func('x, y, z=0'), 1, x=1) + assert check_valid(f) is False + assert check_valid(f, z=3) is False + + f = cytoolz.curry(make_func('x, y, *args, z')) + assert check_valid(f) + assert check_valid(f, 0) + assert check_valid(f(1), 0) + assert check_valid(f(1, 2), 0) + assert check_valid(f(1, 2, 3), 0) + assert check_valid(f(1, 2, 3, 4), 0) + assert check_valid(f(1, 2, 3, 4), z=4) + assert check_valid(f(x=1)) + assert check_valid(f(x=1), 1) is False + assert check_valid(f(x=1), y=2) + + +def test_introspect_curry_partial_py3(): + test_introspect_curry_valid_py3(check_valid=is_partial_args, incomplete=True) + + +def test_introspect_curry_py3(): + if not PY3: + return + f = cytoolz.curry(make_func('')) + assert num_required_args(f) == 0 + assert is_arity(0, f) + assert has_varargs(f) is False + assert has_keywords(f) is False + + f = cytoolz.curry(make_func('x')) + assert num_required_args(f) == 0 + assert is_arity(0, f) is False + assert is_arity(1, f) is False + assert has_varargs(f) is False + assert has_keywords(f) # A side-effect of being curried + + f = cytoolz.curry(make_func('x, y, z=0')) + assert num_required_args(f) == 0 + assert is_arity(0, f) is False + assert is_arity(1, f) is False + assert is_arity(2, f) is False + assert is_arity(3, f) is False + assert has_varargs(f) is False + assert has_keywords(f) + + f = cytoolz.curry(make_func('*args, **kwargs')) + assert num_required_args(f) == 0 + assert has_varargs(f) + assert has_keywords(f) + + +def test_introspect_builtin_modules(): + mods = [builtins, functools, itertools, operator, cytoolz, + cytoolz.functoolz, cytoolz.itertoolz, cytoolz.dicttoolz, cytoolz.recipes] + + blacklist = set() + + def add_blacklist(mod, attr): + if hasattr(mod, attr): + blacklist.add(getattr(mod, attr)) + + add_blacklist(builtins, 'basestring') + add_blacklist(builtins, 'NoneType') + add_blacklist(builtins, '__metaclass__') + add_blacklist(builtins, 'sequenceiterator') + + def is_missing(modname, name, func): + if name.startswith('_') and not name.startswith('__'): + return False + try: + if issubclass(func, BaseException): + return False + except TypeError: + pass + try: + return (callable(func) + and func.__module__ is not None + and modname in func.__module__ + and is_partial_args(func, (), {}) is not True + and func not in blacklist) + except AttributeError: + return False + + missing = {} + for mod in mods: + modname = mod.__name__ + for name, func in vars(mod).items(): + if is_missing(modname, name, func): + if modname not in missing: + missing[modname] = [] + missing[modname].append(name) + if missing: + messages = [] + for modname, names in sorted(missing.items()): + msg = '{0}:\n {1}'.format(modname, '\n '.join(sorted(names))) + messages.append(msg) + message = 'Missing introspection for the following callables:\n\n' + raise AssertionError(message + '\n\n'.join(messages)) + + +def test_inspect_signature_property(): + if not PY3: + return + + # By adding AddX to our signature registry, we can inspect the class + # itself and objects of the class. `inspect.signature` doesn't like + # it when `obj.__signature__` is a property. + class AddX(object): + def __init__(self, func): + self.func = func + + def __call__(self, addx, *args, **kwargs): + return addx + self.func(*args, **kwargs) + + @property + def __signature__(self): + sig = inspect.signature(self.func) + params = list(sig.parameters.values()) + kind = inspect.Parameter.POSITIONAL_OR_KEYWORD + newparam = inspect.Parameter('addx', kind) + params = [newparam] + params + return sig.replace(parameters=params) + + addx = AddX(lambda x: x) + sig = inspect.signature(addx) + assert sig == inspect.Signature(parameters=[ + inspect.Parameter('addx', inspect.Parameter.POSITIONAL_OR_KEYWORD), + inspect.Parameter('x', inspect.Parameter.POSITIONAL_OR_KEYWORD)]) + + assert num_required_args(AddX) is False + _sigs.signatures[AddX] = (_sigs.expand_sig((0, lambda func: None)),) + assert num_required_args(AddX) == 1 + del _sigs.signatures[AddX] + + +def test_inspect_wrapped_property(): + class Wrapped(object): + def __init__(self, func): + self.func = func + + def __call__(self, *args, **kwargs): + return self.func(*args, **kwargs) + + @property + def __wrapped__(self): + return self.func + + func = lambda x: x + wrapped = Wrapped(func) + if PY3: + assert inspect.signature(func) == inspect.signature(wrapped) + + assert num_required_args(Wrapped) == (False if PY33 else None) + _sigs.signatures[Wrapped] = (_sigs.expand_sig((0, lambda func: None)),) + assert num_required_args(Wrapped) == 1 diff --git a/cytoolz/tests/test_serialization.py b/cytoolz/tests/test_serialization.py index 9d2464f..39686f6 100644 --- a/cytoolz/tests/test_serialization.py +++ b/cytoolz/tests/test_serialization.py @@ -1,4 +1,5 @@ from cytoolz import * +import cytoolz import pickle @@ -28,3 +29,14 @@ def test_complement(): g = pickle.loads(pickle.dumps(f)) assert f(True) == g(True) assert f(False) == g(False) + + +def test_instanceproperty(): + p = cytoolz.functoolz.InstanceProperty(bool) + assert p.__get__(None) is None + assert p.__get__(0) is False + assert p.__get__(1) is True + p2 = pickle.loads(pickle.dumps(p)) + assert p2.__get__(None) is None + assert p2.__get__(0) is False + assert p2.__get__(1) is True diff --git a/cytoolz/tests/test_signatures.py b/cytoolz/tests/test_signatures.py index d395d08..ecbb002 100644 --- a/cytoolz/tests/test_signatures.py +++ b/cytoolz/tests/test_signatures.py @@ -1,12 +1,10 @@ import functools -import sys -from cytoolz._signatures import (builtins, is_builtin_valid_args, - is_builtin_partial_args) +import cytoolz._signatures as _sigs +from cytoolz._signatures import builtins, _is_valid_args, _is_partial_args from cytoolz.compatibility import PY3 -from cytoolz.utils import raises -def test_is_valid(check_valid=is_builtin_valid_args, incomplete=False): +def test_is_valid(check_valid=_is_valid_args, incomplete=False): orig_check_valid = check_valid check_valid = lambda func, *args, **kwargs: orig_check_valid(func, args, kwargs) @@ -77,5 +75,13 @@ def test_is_valid(check_valid=is_builtin_valid_args, incomplete=False): def test_is_partial(): - test_is_valid(check_valid=is_builtin_partial_args, incomplete=True) + test_is_valid(check_valid=_is_partial_args, incomplete=True) + + +def test_for_coverage(): # :) + assert _sigs._is_arity(1, 1) is None + assert _sigs._is_arity(1, all) + assert _sigs._has_varargs(None) is None + assert _sigs._has_keywords(None) is None + assert _sigs._num_required_args(None) is None From df5333b7328cffd99dd6f8c036cd6d937d41511c Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 30 May 2016 13:49:36 -0500 Subject: [PATCH 093/146] Add `toolz` as a dependency in setup.py. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 5b31877..d4f4e7e 100644 --- a/setup.py +++ b/setup.py @@ -109,5 +109,6 @@ 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Utilities', ], + install_requires=['toolz >= 0.8.0'], # zip_safe=False ) From c9e4fe0c73acf0e2d4669265f88c3f2e46da04dd Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 30 May 2016 16:01:06 -0500 Subject: [PATCH 094/146] Update so `cytoolz` passes release tests. Change to use setuptools. --- Makefile | 6 +++- cytoolz/curried/__init__.py | 70 +++++++++++++++++++------------------ cytoolz/curried/operator.py | 25 +++++-------- cytoolz/dicttoolz.pyx | 28 +++++++-------- cytoolz/functoolz.pyx | 42 +++++----------------- cytoolz/itertoolz.pyx | 2 -- etc/generate_curried.py | 2 +- setup.py | 12 ++++--- 8 files changed, 80 insertions(+), 107 deletions(-) diff --git a/Makefile b/Makefile index efa19b3..c5f0813 100644 --- a/Makefile +++ b/Makefile @@ -7,4 +7,8 @@ test: inplace nosetests -s --with-doctest cytoolz/ clean: - rm cytoolz/*.c cytoolz/*.so + rm -f cytoolz/*.c cytoolz/*.so cytoolz/*/*.c cytoolz/*/*.so + rm -rf build/ cytoolz/__pycache__/ cytoolz/*/__pycache__/ + +curried: + $(PYTHON) etc/generate_curried.py > cytoolz/curried/__init__.py diff --git a/cytoolz/curried/__init__.py b/cytoolz/curried/__init__.py index ecb71a7..174c9a0 100644 --- a/cytoolz/curried/__init__.py +++ b/cytoolz/curried/__init__.py @@ -35,50 +35,52 @@ _curry_set = frozenset([ - cytoolz.nth, - cytoolz.partition, - cytoolz.take_nth, - cytoolz.tail, - cytoolz.valfilter, - cytoolz.memoize, - cytoolz.reduceby, - cytoolz.topk, - cytoolz.join, + cytoolz.accumulate, + cytoolz.assoc, + cytoolz.assoc_in, + cytoolz.cons, + cytoolz.countby, cytoolz.do, - cytoolz.sorted, - cytoolz.interpose, - cytoolz.take, - cytoolz.pluck, cytoolz.drop, + cytoolz.excepts, + cytoolz.filter, + cytoolz.flip, + cytoolz.get, cytoolz.get_in, - cytoolz.reduce, - cytoolz.itemfilter, - cytoolz.accumulate, - cytoolz.merge, + cytoolz.groupby, cytoolz.interleave, + cytoolz.interpose, + cytoolz.itemfilter, + cytoolz.itemmap, cytoolz.iterate, - cytoolz.get, - cytoolz.remove, - cytoolz.valmap, + cytoolz.join, + cytoolz.keyfilter, cytoolz.keymap, - cytoolz.cons, - cytoolz.unique, - cytoolz.partitionby, - cytoolz.itemmap, - cytoolz.sliding_window, cytoolz.map, - cytoolz.partition_all, - cytoolz.assoc, cytoolz.mapcat, - cytoolz.filter, - cytoolz.countby, + cytoolz.memoize, + cytoolz.merge, cytoolz.merge_with, - cytoolz.update_in, - cytoolz.keyfilter, - cytoolz.groupby, - cytoolz.assoc_in, + cytoolz.nth, + cytoolz.partial, + cytoolz.partition, + cytoolz.partition_all, + cytoolz.partitionby, + cytoolz.pluck, cytoolz.random_sample, - cytoolz.excepts, + cytoolz.reduce, + cytoolz.reduceby, + cytoolz.remove, + cytoolz.sliding_window, + cytoolz.sorted, + cytoolz.tail, + cytoolz.take, + cytoolz.take_nth, + cytoolz.topk, + cytoolz.unique, + cytoolz.update_in, + cytoolz.valfilter, + cytoolz.valmap, ]) diff --git a/cytoolz/curried/operator.py b/cytoolz/curried/operator.py index 0eda560..d9e5e47 100644 --- a/cytoolz/curried/operator.py +++ b/cytoolz/curried/operator.py @@ -2,29 +2,22 @@ import operator -from cytoolz import curry +from cytoolz.functoolz import curry, num_required_args, has_keywords -# We use a blacklist instead of whitelist because: -# 1. We have more things to include than exclude. -# 2. This gives us access to things like matmul iff we are in Python >=3.5. -no_curry = frozenset(( - 'abs', - 'index', - 'inv', - 'invert', - 'neg', - 'not_', - 'pos', - 'truth', -)) +def should_curry(f): + num = num_required_args(f) + return num is None or num > 1 or num == 1 and has_keywords(f) is not False + locals().update( - dict((name, curry(f) if name not in no_curry else f) + dict((name, curry(f) if should_curry(f) else f) for name, f in vars(operator).items() if callable(f)), ) # Clean up the namespace. del curry -del no_curry +del num_required_args +del has_keywords del operator +del should_curry diff --git a/cytoolz/dicttoolz.pyx b/cytoolz/dicttoolz.pyx index a06a585..35c2f39 100644 --- a/cytoolz/dicttoolz.pyx +++ b/cytoolz/dicttoolz.pyx @@ -1,14 +1,12 @@ from cpython.dict cimport (PyDict_Check, PyDict_CheckExact, PyDict_GetItem, PyDict_Merge, PyDict_New, PyDict_Next, PyDict_SetItem, PyDict_Update, PyDict_DelItem) -from cpython.exc cimport PyErr_Clear, PyErr_GivenExceptionMatches, PyErr_Occurred from cpython.list cimport PyList_Append, PyList_New from cpython.object cimport PyObject_SetItem from cpython.ref cimport PyObject, Py_DECREF, Py_INCREF, Py_XDECREF -#from cpython.type cimport PyType_Check # Locally defined bindings that differ from `cython.cpython` bindings -from cytoolz.cpython cimport PtrObject_GetItem, PyDict_Next_Compat, PtrIter_Next +from cytoolz.cpython cimport PyDict_Next_Compat, PtrIter_Next from copy import copy @@ -436,6 +434,8 @@ def dissoc(d, *keys): {'x': 1} >>> dissoc({'x': 1, 'y': 2}, 'y', 'x') {} + >>> dissoc({'x': 1}, 'y') # Ignores missing keys + {'x': 1} """ return c_dissoc(d, keys) @@ -534,7 +534,7 @@ cpdef object get_in(object keys, object coll, object default=None, object no_def >>> get_in(['purchase', 'items', 10], transaction) >>> get_in(['purchase', 'total'], transaction, 0) 0 - >>> get_in(['y'], {}, no_default=True) # doctest: +SKIP + >>> get_in(['y'], {}, no_default=True) Traceback (most recent call last): ... KeyError: 'y' @@ -544,16 +544,12 @@ cpdef object get_in(object keys, object coll, object default=None, object no_def operator.getitem """ cdef object item - cdef PyObject *obj - for item in keys: - obj = PtrObject_GetItem(coll, item) - if obj is NULL: - item = PyErr_Occurred() - PyErr_Clear() - if no_default or not PyErr_GivenExceptionMatches(item, _get_in_exceptions): - raise item - return default - Py_XDECREF(obj) - coll = obj - return coll + try: + for item in keys: + coll = coll[item] + return coll + except _get_in_exceptions: + if no_default: + raise + return default diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index bf9433b..a252aba 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -12,17 +12,13 @@ from toolz.functoolz import (InstanceProperty, instanceproperty, is_arity, is_valid_args, is_partial_args) from cpython.dict cimport PyDict_Merge, PyDict_New -from cpython.exc cimport PyErr_Clear, PyErr_Occurred, PyErr_GivenExceptionMatches from cpython.object cimport (PyCallable_Check, PyObject_Call, PyObject_CallObject, PyObject_RichCompare, Py_EQ, Py_NE) -from cpython.ref cimport PyObject, Py_DECREF +from cpython.ref cimport PyObject from cpython.sequence cimport PySequence_Concat from cpython.set cimport PyFrozenSet_New from cpython.tuple cimport PyTuple_Check, PyTuple_GET_SIZE -# Locally defined bindings that differ from `cython.cpython` bindings -from cytoolz.cpython cimport PtrObject_Call - __all__ = ['identity', 'thread_first', 'thread_last', 'memoize', 'compose', 'pipe', 'complement', 'juxt', 'do', 'curry', 'memoize', 'flip', @@ -221,7 +217,6 @@ cdef class curry: return PyObject_RichCompare(id(self), id(other), op) def __call__(self, *args, **kwargs): - cdef PyObject *obj cdef object val if PyTuple_GET_SIZE(args) == 0: @@ -230,20 +225,12 @@ cdef class curry: args = PySequence_Concat(self.args, args) if self.keywords is not None: PyDict_Merge(kwargs, self.keywords, False) - - obj = PtrObject_Call(self.func, args, kwargs) - if obj is not NULL: - val = obj - Py_DECREF(val) - return val - - val = PyErr_Occurred() - PyErr_Clear() - if (PyErr_GivenExceptionMatches(val, TypeError) and - self._should_curry_internal(args, kwargs, val) - ): - return type(self)(self.func, *args, **kwargs) - raise val + try: + return self.func(*args, **kwargs) + except TypeError as val: + if self._should_curry_internal(args, kwargs, val): + return type(self)(self.func, *args, **kwargs) + raise def _should_curry_internal(self, args, kwargs, exc=None): func = self.func @@ -278,7 +265,6 @@ cdef class curry: return type(self)(self, *args, **kwargs) def call(self, *args, **kwargs): - cdef PyObject *obj cdef object val if PyTuple_GET_SIZE(args) == 0: @@ -287,16 +273,7 @@ cdef class curry: args = PySequence_Concat(self.args, args) if self.keywords is not None: PyDict_Merge(kwargs, self.keywords, False) - - obj = PtrObject_Call(self.func, args, kwargs) - if obj is not NULL: - val = obj - Py_DECREF(val) - return val - - val = PyErr_Occurred() - PyErr_Clear() - raise val + return self.func(*args, **kwargs) def __get__(self, instance, owner): if instance is None: @@ -605,7 +582,7 @@ cdef object c_juxt(object funcs): def juxt(*funcs): """ - Creates a function that calls several functions with the same arguments. + Creates a function that calls several functions with the same arguments Takes several functions and returns a function that applies its arguments to each of those functions then returns a tuple of the results. @@ -647,7 +624,6 @@ cpdef object do(object func, object x): 12 >>> log [1, 11] - """ func(x) return x diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 56ec276..08d37ba 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -888,7 +888,6 @@ cdef class iterate: 4 >>> next(powers_of_two) 8 - """ def __cinit__(self, object func, object x): self.func = func @@ -1683,7 +1682,6 @@ cpdef object peek(object seq): 0 >>> list(seq) [0, 1, 2, 3, 4] - """ iterator = iter(seq) item = next(iterator) diff --git a/etc/generate_curried.py b/etc/generate_curried.py index e6114b2..e4c164f 100755 --- a/etc/generate_curried.py +++ b/etc/generate_curried.py @@ -74,7 +74,7 @@ def _curry_namespace(ns): ct = vars(cytoolz) return ( 'cytoolz.' + name + ',' - for name, f in ns.items() if isinstance(f, toolz.curry) and name in ct + for name, f in sorted(ns.items()) if isinstance(f, toolz.curry) and name in ct ) diff --git a/setup.py b/setup.py index d4f4e7e..db18b29 100644 --- a/setup.py +++ b/setup.py @@ -15,8 +15,12 @@ """ import os.path import sys -from distutils.core import setup -from distutils.extension import Extension +try: + from setuptools import setup + from setuptools import Extension +except ImportError: + from distutils.core import setup + from distutils.extension import Extension info = {} filename = os.path.join('cytoolz', '_version.py') @@ -57,7 +61,7 @@ ext_modules = [] for modname in ['dicttoolz', 'functoolz', 'itertoolz', 'curried/exceptions', 'recipes', 'utils']: - ext_modules.append(Extension('cytoolz.' + modname, + ext_modules.append(Extension('cytoolz.' + modname.replace('/', '.'), ['cytoolz/' + modname + suffix])) if use_cython: @@ -110,5 +114,5 @@ 'Topic :: Utilities', ], install_requires=['toolz >= 0.8.0'], - # zip_safe=False + zip_safe=False, ) From 2bfd24227ffd2994dfd781ffe22ddaed4ddeae47 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 30 May 2016 18:12:27 -0500 Subject: [PATCH 095/146] Only use setuptools, 'cause it's 2016 fer cryin' out loud. Also, update `curried.exceptions`. --- cytoolz/curried/exceptions.py | 17 +++++++++++++++ cytoolz/curried/exceptions.pyx | 40 ---------------------------------- setup.py | 10 ++------- 3 files changed, 19 insertions(+), 48 deletions(-) create mode 100644 cytoolz/curried/exceptions.py delete mode 100644 cytoolz/curried/exceptions.pyx diff --git a/cytoolz/curried/exceptions.py b/cytoolz/curried/exceptions.py new file mode 100644 index 0000000..ff172dd --- /dev/null +++ b/cytoolz/curried/exceptions.py @@ -0,0 +1,17 @@ +from cytoolz import curry, merge as _merge, merge_with as _merge_with + +__all__ = ['merge', 'merge_with'] + + +@curry +def merge(d, *dicts, **kwargs): + return _merge(d, *dicts, **kwargs) + + +@curry +def merge_with(func, d, *dicts, **kwargs): + return _merge_with(func, d, *dicts, **kwargs) + + +merge.__doc__ = _merge.__doc__ +merge_with.__doc__ = _merge_with.__doc__ diff --git a/cytoolz/curried/exceptions.pyx b/cytoolz/curried/exceptions.pyx deleted file mode 100644 index b95cdf6..0000000 --- a/cytoolz/curried/exceptions.pyx +++ /dev/null @@ -1,40 +0,0 @@ -from cytoolz import curry - -from cpython.dict cimport PyDict_Check -from cytoolz.dicttoolz cimport c_merge, c_merge_with - -__all__ = ['merge', 'merge_with'] - - -@curry -def merge(*dicts, **kwargs): - if len(dicts) == 0: - raise TypeError() - if len(dicts) == 1 and not PyDict_Check(dicts[0]): - dicts = dicts[0] - return c_merge(dicts) - - -@curry -def merge_with(func, *dicts, **kwargs): - """ - Merge dictionaries and apply function to combined values - - A key may occur in more than one dict, and all values mapped from the key - will be passed to the function as a list, such as func([val1, val2, ...]). - - >>> merge_with(sum, {1: 1, 2: 2}, {1: 10, 2: 20}) - {1: 11, 2: 22} - - >>> merge_with(first, {1: 1, 2: 2}, {2: 20, 3: 30}) # doctest: +SKIP - {1: 1, 2: 2, 3: 30} - - See Also: - merge - """ - if len(dicts) == 0: - raise TypeError() - if len(dicts) == 1 and not PyDict_Check(dicts[0]): - dicts = dicts[0] - - return c_merge_with(func, dicts) diff --git a/setup.py b/setup.py index db18b29..ec0cc9a 100644 --- a/setup.py +++ b/setup.py @@ -15,12 +15,7 @@ """ import os.path import sys -try: - from setuptools import setup - from setuptools import Extension -except ImportError: - from distutils.core import setup - from distutils.extension import Extension +from setuptools import setup, Extension info = {} filename = os.path.join('cytoolz', '_version.py') @@ -59,8 +54,7 @@ suffix = '.c' ext_modules = [] -for modname in ['dicttoolz', 'functoolz', 'itertoolz', - 'curried/exceptions', 'recipes', 'utils']: +for modname in ['dicttoolz', 'functoolz', 'itertoolz', 'recipes', 'utils']: ext_modules.append(Extension('cytoolz.' + modname.replace('/', '.'), ['cytoolz/' + modname + suffix])) From a957af768251816079caade89bfc89b7b497d4c5 Mon Sep 17 00:00:00 2001 From: Jim Crist Date: Wed, 1 Jun 2016 17:08:29 -0500 Subject: [PATCH 096/146] Use `==` instead of `is` to check for no_default Previously, `is` was used to check if keywords were equivalent to `no_default`. This failed if the default string was serialized using pickle, as then the reconstructed string isn't the same object. This PR fixes that by replacing all is checks for `no_default` with `==`. Also modified the `no_default` string to be equivalent to that in `toolz`. --- cytoolz/itertoolz.pyx | 12 ++++++------ cytoolz/tests/test_itertoolz.py | 17 +++++++++++++++++ cytoolz/utils.pyx | 2 +- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 56ec276..59511de 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -100,7 +100,7 @@ cdef class accumulate: def __next__(self): if self.result is self: - if self.initial is not no_default: + if self.initial != no_default: self.result = self.initial else: self.result = next(self.iter_seq) @@ -651,7 +651,7 @@ cpdef object get(object ind, object seq, object default=no_default): i = PyList_GET_SIZE(ind) result = PyTuple_New(i) # List of indices, no default - if default is no_default: + if default == no_default: for i, val in enumerate(ind): val = seq[val] Py_INCREF(val) @@ -677,7 +677,7 @@ cpdef object get(object ind, object seq, object default=no_default): if obj is NULL: val = PyErr_Occurred() PyErr_Clear() - if default is no_default: + if default == no_default: raise val if PyErr_GivenExceptionMatches(val, _get_exceptions): return default @@ -839,7 +839,7 @@ cpdef dict reduceby(object key, object binop, object seq, object init=no_default cdef dict d = {} cdef object item, keyval cdef Py_ssize_t i, N - cdef bint skip_init = init is no_default + cdef bint skip_init = init == no_default cdef bint call_init = callable(init) if callable(key): for item in seq: @@ -1153,12 +1153,12 @@ cpdef object pluck(object ind, object seqs, object default=no_default): map """ if isinstance(ind, list): - if default is not no_default: + if default != no_default: return _pluck_list_default(ind, seqs, default) if PyList_GET_SIZE(ind) < 10: return _pluck_list(ind, seqs) return map(itemgetter(*ind), seqs) - if default is no_default: + if default == no_default: return _pluck_index(ind, seqs) return _pluck_index_default(ind, seqs, default) diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index c96e911..9568893 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -3,6 +3,7 @@ from cytoolz.utils import raises from functools import partial from random import Random +from pickle import dumps, loads from cytoolz.itertoolz import (remove, groupby, merge_sorted, concat, concatv, interleave, unique, isiterable, getter, @@ -17,6 +18,10 @@ from operator import add, mul +# is comparison will fail between this and no_default +no_default2 = loads(dumps('__no__default__')) + + def identity(x): return x @@ -181,6 +186,7 @@ def test_get(): assert raises(KeyError, lambda: get(10, {'a': 1})) assert raises(TypeError, lambda: get({}, [1, 2, 3])) assert raises(TypeError, lambda: get([1, 2, 3], 1, None)) + assert raises(KeyError, lambda: get('foo', {}, default=no_default2)) def test_mapcat(): @@ -248,6 +254,8 @@ def test_reduceby(): def test_reduce_by_init(): assert reduceby(iseven, add, [1, 2, 3, 4]) == {True: 2 + 4, False: 1 + 3} + assert reduceby(iseven, add, [1, 2, 3, 4], no_default2) == {True: 2 + 4, + False: 1 + 3} def test_reduce_by_callable_default(): @@ -274,6 +282,7 @@ def binop(a, b): start = object() assert list(accumulate(binop, [], start)) == [start] + assert list(accumulate(add, [1, 2, 3], no_default2)) == [1, 3, 6] def test_accumulate_works_on_consumable_iterables(): @@ -328,6 +337,9 @@ def test_pluck(): assert raises(IndexError, lambda: list(pluck(1, [[0]]))) assert raises(KeyError, lambda: list(pluck('name', [{'id': 1}]))) + assert list(pluck(0, [[0, 1], [2, 3], [4, 5]], no_default2)) == [0, 2, 4] + assert raises(IndexError, lambda: list(pluck(1, [[0]], no_default2))) + def test_join(): names = [(1, 'one'), (2, 'two'), (3, 'three')] @@ -345,6 +357,11 @@ def addpair(pair): assert result == expected + result = set(starmap(add, join(first, names, second, fruit, + left_default=no_default2, + right_default=no_default2))) + assert result == expected + def test_getter(): assert getter(0)('Alice') == 'A' diff --git a/cytoolz/utils.pyx b/cytoolz/utils.pyx index 20a20ef..8d39165 100644 --- a/cytoolz/utils.pyx +++ b/cytoolz/utils.pyx @@ -19,7 +19,7 @@ try: # Attempt to get the no_default sentinel object from toolz from toolz.utils import no_default except ImportError: - no_default = '__no_default__' + no_default = '__no__default__' def include_dirs(): From 1a7f33fda9f42c62a18bdcce6ba3f6c02567f584 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 2 Jun 2016 18:35:31 -0500 Subject: [PATCH 097/146] Bump to 0.8.0. --- conda.recipe/meta.yaml | 2 +- cytoolz/_version.py | 4 ++-- cytoolz/curried/exceptions.py | 15 ++++++++------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index a3a3e11..e5ddf0a 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: cytoolz - version: "0.7.4" + version: "0.8.0" build: number: {{environ.get('BINSTAR_BUILD', 1)}} diff --git a/cytoolz/_version.py b/cytoolz/_version.py index 7e05c3a..65ff559 100644 --- a/cytoolz/_version.py +++ b/cytoolz/_version.py @@ -1,2 +1,2 @@ -__version__ = '0.7.6dev' -__toolz_version__ = '0.7.4' +__version__ = '0.8.0' +__toolz_version__ = '0.8.0' diff --git a/cytoolz/curried/exceptions.py b/cytoolz/curried/exceptions.py index ff172dd..ce09b3d 100644 --- a/cytoolz/curried/exceptions.py +++ b/cytoolz/curried/exceptions.py @@ -1,17 +1,18 @@ -from cytoolz import curry, merge as _merge, merge_with as _merge_with +import cytoolz +#from cytoolz import curry, merge as _merge, merge_with as _merge_with __all__ = ['merge', 'merge_with'] -@curry +@cytoolz.curry def merge(d, *dicts, **kwargs): - return _merge(d, *dicts, **kwargs) + return cytoolz.merge(d, *dicts, **kwargs) -@curry +@cytoolz.curry def merge_with(func, d, *dicts, **kwargs): - return _merge_with(func, d, *dicts, **kwargs) + return cytoolz.merge_with(func, d, *dicts, **kwargs) -merge.__doc__ = _merge.__doc__ -merge_with.__doc__ = _merge_with.__doc__ +merge.__doc__ = cytoolz.merge.__doc__ +merge_with.__doc__ = cytoolz.merge_with.__doc__ From 003cf6e3a412434cbf3a3d78992361cd0a5515ef Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 2 Jun 2016 19:01:40 -0500 Subject: [PATCH 098/146] 0.8.0 just released. Bump to next dev version. --- cytoolz/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cytoolz/_version.py b/cytoolz/_version.py index 65ff559..7805a97 100644 --- a/cytoolz/_version.py +++ b/cytoolz/_version.py @@ -1,2 +1,2 @@ -__version__ = '0.8.0' +__version__ = '0.8.1dev' __toolz_version__ = '0.8.0' From 6df45d2829d56ff0ac29335f6785fd03bee1164c Mon Sep 17 00:00:00 2001 From: Jim Crist Date: Mon, 3 Oct 2016 11:16:00 -0500 Subject: [PATCH 099/146] Add a copytests command to the Makefile Copies the tests over from pytoolz, assuming toolz is in the parent directory. --- Makefile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Makefile b/Makefile index c5f0813..ae22ed3 100644 --- a/Makefile +++ b/Makefile @@ -12,3 +12,14 @@ clean: curried: $(PYTHON) etc/generate_curried.py > cytoolz/curried/__init__.py + +copytests: + for f in ../toolz/toolz/tests/test*py; \ + do \ + if [[ $$f == *test_utils* ]]; then continue ; fi; \ + newf=`echo $$f | sed 's/...toolz.toolz/cytoolz/g'`; \ + sed -e 's/toolz/cytoolz/g' -e 's/itercytoolz/itertoolz/' \ + -e 's/dictcytoolz/dicttoolz/g' -e 's/funccytoolz/functoolz/g' \ + $$f > $$newf; \ + echo $$f $$newf; \ + done From 0c37ae625b33d049fc7744744165cdcf65673764 Mon Sep 17 00:00:00 2001 From: Jim Crist Date: Mon, 3 Oct 2016 11:16:25 -0500 Subject: [PATCH 100/146] Fix bug in pluck with default The non-list version of pluck with default failed to set actually set `default` as an attribute of the object, resulting in all accesses being None. This passed the tests, since the only time a default was used was when it was None. --- cytoolz/itertoolz.pyx | 1 + cytoolz/tests/test_itertoolz.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 3b96480..639bcd0 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -1054,6 +1054,7 @@ cdef class _pluck_index_default: def __cinit__(self, object ind, object seqs, object default): self.ind = ind self.iterseqs = iter(seqs) + self.default = default def __iter__(self): return self diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index 9568893..9f35716 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -328,11 +328,10 @@ def test_pluck(): data = [{'id': 1, 'name': 'cheese'}, {'id': 2, 'name': 'pies', 'price': 1}] assert list(pluck('id', data)) == [1, 2] - assert list(pluck('price', data, None)) == [None, 1] + assert list(pluck('price', data, 0)) == [0, 1] assert list(pluck(['id', 'name'], data)) == [(1, 'cheese'), (2, 'pies')] assert list(pluck(['name'], data)) == [('cheese',), ('pies',)] - assert list(pluck(['price', 'other'], data, None)) == [(None, None), - (1, None)] + assert list(pluck(['price', 'other'], data, 0)) == [(0, 0), (1, 0)] assert raises(IndexError, lambda: list(pluck(1, [[0]]))) assert raises(KeyError, lambda: list(pluck('name', [{'id': 1}]))) From 68140f5d09feec0b8b9e3fdae00876541a1e7f82 Mon Sep 17 00:00:00 2001 From: Adam Chainz Date: Mon, 10 Oct 2016 08:24:40 +0100 Subject: [PATCH 101/146] Convert readthedocs links for their .org -> .io migration for hosted projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As per [their blog post of the 27th April](https://blog.readthedocs.com/securing-subdomains/) ‘Securing subdomains’: > Starting today, Read the Docs will start hosting projects from subdomains on the domain readthedocs.io, instead of on readthedocs.org. This change addresses some security concerns around site cookies while hosting user generated data on the same domain as our dashboard. Test Plan: Manually visited all the links I’ve modified. --- README.rst | 4 ++-- conda.recipe/meta.yaml | 2 +- cytoolz/functoolz.pyx | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 42df745..6af7dc3 100644 --- a/README.rst +++ b/README.rst @@ -26,8 +26,8 @@ projects developed in Cython. Since ``toolz`` is able to process very large (potentially infinite) data sets, the performance increase gained by using ``cytoolz`` can be significant. -See the PyToolz documentation at http://toolz.readthedocs.org and the full -`API Documentation `__ +See the PyToolz documentation at https://toolz.readthedocs.io and the full +`API Documentation `__ for more details. LICENSE diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index e5ddf0a..c7eb3fc 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -28,5 +28,5 @@ test: - py.test -x --doctest-modules --pyargs cytoolz about: - home: http://toolz.readthedocs.org/ + home: https://toolz.readthedocs.io/ license: BSD diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index a252aba..3ce9e26 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -164,7 +164,7 @@ cdef class curry: See Also: cytoolz.curried - namespace of curried functions - http://toolz.readthedocs.org/en/latest/curry.html + https://toolz.readthedocs.io/en/latest/curry.html """ def __cinit__(self, *args, **kwargs): From 81a30fd6fba9105f4d50ae000dacdd9d64cdc729 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 20 Oct 2016 11:14:41 -0400 Subject: [PATCH 102/146] MANIFEST.in: Add the license file. --- MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index d0ccc69..2a554c5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,3 @@ +include LICENSE.txt + include cytoolz/tests/*.py From 5fe80792a2960fb3da6d085c660f2b552e10744f Mon Sep 17 00:00:00 2001 From: Adam Chidlow Date: Wed, 26 Oct 2016 17:57:44 +0800 Subject: [PATCH 103/146] Fix issue #85: declare tail in itertoolz.pxd --- cytoolz/itertoolz.pxd | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index 900787f..546fc83 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -58,6 +58,9 @@ cpdef object isdistinct(object seq) cpdef object take(Py_ssize_t n, object seq) +cpdef object tail(Py_ssize_t n, object seq) + + cpdef object drop(Py_ssize_t n, object seq) From 7addee72b25970d8bf8a6ff4954e98a28ea9f4dc Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 30 Oct 2016 23:20:27 -0500 Subject: [PATCH 104/146] Cython >=0.25 now uses `get_directive_defaults` instead of `directive_defaults`. Updated in a compatibile way. --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ec0cc9a..014844a 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,12 @@ ['cytoolz/' + modname + suffix])) if use_cython: - from Cython.Compiler.Options import directive_defaults + try: + from Cython.Compiler.Options import get_directive_defaults + directive_defaults = get_directive_defaults() + except ImportError: + # for Cython < 0.25 + from Cython.Compiler.Options import directive_defaults directive_defaults['embedsignature'] = True directive_defaults['binding'] = True ext_modules = cythonize(ext_modules) From 88c65e9b468da1aaee12093800e0fb1bc92f91f8 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 31 Oct 2016 00:47:01 -0500 Subject: [PATCH 105/146] Change function signature introspection tests to support Cython 0.25. --- cytoolz/__init__.pxd | 2 +- cytoolz/_signatures.py | 6 +- cytoolz/functoolz.pxd | 6 +- cytoolz/functoolz.pyx | 112 +++++++++++++++++----------- cytoolz/itertoolz.pyx | 22 +++--- cytoolz/tests/test_embedded_sigs.py | 12 ++- cytoolz/tests/test_none_safe.py | 2 +- 7 files changed, 97 insertions(+), 65 deletions(-) diff --git a/cytoolz/__init__.pxd b/cytoolz/__init__.pxd index 87b3a7b..88cd4bf 100644 --- a/cytoolz/__init__.pxd +++ b/cytoolz/__init__.pxd @@ -8,7 +8,7 @@ from cytoolz.itertoolz cimport ( from cytoolz.functoolz cimport ( c_compose, c_juxt, c_memoize, c_pipe, c_thread_first, c_thread_last, - complement, curry, do, identity, memoize, excepts) + complement, curry, do, identity, excepts, c_flip) from cytoolz.dicttoolz cimport ( diff --git a/cytoolz/_signatures.py b/cytoolz/_signatures.py index 0f84632..d7fb24e 100644 --- a/cytoolz/_signatures.py +++ b/cytoolz/_signatures.py @@ -36,8 +36,8 @@ cytoolz_info['cytoolz.functoolz'] = dict( Compose=[ lambda *funcs: None], - _flip=[ - lambda f, a, b: None], + c_flip=[ + lambda func, a, b: None], c_memoize=[ lambda func, cache=None, key=None: None], complement=[ @@ -55,7 +55,7 @@ juxt=[ lambda *funcs: None], memoize=[ - lambda func=None, cache=None, key=None: None], + lambda func, cache=None, key=None: None], pipe=[ lambda data, *funcs: None], return_none=[ diff --git a/cytoolz/functoolz.pxd b/cytoolz/functoolz.pxd index 5e6d2f4..7c7a515 100644 --- a/cytoolz/functoolz.pxd +++ b/cytoolz/functoolz.pxd @@ -24,9 +24,6 @@ cdef class c_memoize: cdef bint may_have_kwargs -cpdef object memoize(object func=*, object cache=*, object key=*) - - cdef class Compose: cdef public object first cdef public tuple funcs @@ -52,6 +49,9 @@ cdef object c_juxt(object funcs) cpdef object do(object func, object x) +cpdef object c_flip(object func, object a, object b) + + cpdef object return_none(object exc) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index a252aba..1071693 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -11,6 +11,7 @@ from toolz.functoolz import (InstanceProperty, instanceproperty, is_arity, num_required_args, has_varargs, has_keywords, is_valid_args, is_partial_args) +cimport cython from cpython.dict cimport PyDict_Merge, PyDict_New from cpython.object cimport (PyCallable_Check, PyObject_Call, PyObject_CallObject, PyObject_RichCompare, Py_EQ, Py_NE) @@ -326,6 +327,44 @@ cdef class curry: cdef class c_memoize: + """ memoize(func, cache=None, key=None) + + Cache a function's result for speedy future evaluation + + Considerations: + Trades memory for speed. + Only use on pure functions. + + >>> def add(x, y): return x + y + >>> add = memoize(add) + + Or use as a decorator + + >>> @memoize + ... def add(x, y): + ... return x + y + + Use the ``cache`` keyword to provide a dict-like object as an initial cache + + >>> @memoize(cache={(1, 2): 3}) + ... def add(x, y): + ... return x + y + + Note that the above works as a decorator because ``memoize`` is curried. + + It is also possible to provide a ``key(args, kwargs)`` function that + calculates keys used for the cache, which receives an ``args`` tuple and + ``kwargs`` dict as input, and must return a hashable value. However, + the default key function should be sufficient most of the time. + + >>> # Use key function that ignores extraneous keyword arguments + >>> @memoize(key=lambda args, kwargs: args) + ... def add(x, y, verbose=False): + ... if verbose: + ... print('Calculating %s + %s' % (x, y)) + ... return x + y + """ + property __doc__: def __get__(self): return self.func.__doc__ @@ -379,47 +418,7 @@ cdef class c_memoize: return curry(self, instance) -cpdef object memoize(object func=None, object cache=None, object key=None): - """ - Cache a function's result for speedy future evaluation - - Considerations: - Trades memory for speed. - Only use on pure functions. - - >>> def add(x, y): return x + y - >>> add = memoize(add) - - Or use as a decorator - - >>> @memoize - ... def add(x, y): - ... return x + y - - Use the ``cache`` keyword to provide a dict-like object as an initial cache - - >>> @memoize(cache={(1, 2): 3}) - ... def add(x, y): - ... return x + y - - Note that the above works as a decorator because ``memoize`` is curried. - - It is also possible to provide a ``key(args, kwargs)`` function that - calculates keys used for the cache, which receives an ``args`` tuple and - ``kwargs`` dict as input, and must return a hashable value. However, - the default key function should be sufficient most of the time. - - >>> # Use key function that ignores extraneous keyword arguments - >>> @memoize(key=lambda args, kwargs: args) - ... def add(x, y, verbose=False): - ... if verbose: - ... print('Calculating %s + %s' % (x, y)) - ... return x + y - """ - # pseudo-curry - if func is None: - return curry(c_memoize, cache=cache, key=key) - return c_memoize(func, cache=cache, key=key) +memoize = curry(c_memoize) cdef class Compose: @@ -629,11 +628,36 @@ cpdef object do(object func, object x): return x -cpdef object _flip(object f, object a, object b): - return PyObject_CallObject(f, (b, a)) +@cython.embedsignature(False) +cpdef object c_flip(object func, object a, object b): + """ flip(func, a, b) + + Call the function call with the arguments flipped + + This function is curried. + + >>> def div(a, b): + ... return a / b + ... + >>> flip(div, 2, 1) + 0.5 + >>> div_by_two = flip(div, 2) + >>> div_by_two(4) + 2.0 + + This is particularly useful for built in functions and functions defined + in C extensions that accept positional only arguments. For example: + isinstance, issubclass. + + >>> data = [1, 'a', 'b', 2, 1.5, object(), 3] + >>> only_ints = list(filter(flip(isinstance, int), data)) + >>> only_ints + [1, 2, 3] + """ + return PyObject_CallObject(func, (b, a)) -flip = curry(_flip) +flip = curry(c_flip) cpdef object return_none(object exc): diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 639bcd0..9ac5c36 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -89,7 +89,7 @@ cdef class accumulate: See Also: itertools.accumulate : In standard itertools for Python 3.2+ """ - def __cinit__(self, object binop, object seq, object initial=no_default): + def __cinit__(self, object binop, object seq, object initial='__no__default__'): self.binop = binop self.iter_seq = iter(seq) self.result = self # sentinel @@ -413,7 +413,7 @@ cdef class _unique_identity: return item -cpdef object unique(object seq, object key=identity): +cpdef object unique(object seq, object key=None): """ Return only unique elements of a sequence @@ -427,7 +427,7 @@ cpdef object unique(object seq, object key=identity): >>> tuple(unique(['cat', 'mouse', 'dog', 'hen'], key=len)) ('cat', 'mouse') """ - if key is identity: + if key is None: return _unique_identity(seq) else: return _unique_key(seq, key) @@ -594,7 +594,7 @@ cpdef object last(object seq): val = no_default for val in seq: pass - if val is no_default: + if val == no_default: raise IndexError return val @@ -609,7 +609,7 @@ cdef tuple _get_exceptions = (IndexError, KeyError, TypeError) cdef tuple _get_list_exc = (IndexError, KeyError) -cpdef object get(object ind, object seq, object default=no_default): +cpdef object get(object ind, object seq, object default='__no__default__'): """ Get element in a sequence or dict @@ -774,7 +774,7 @@ cdef inline object _reduceby_core(dict d, object key, object item, object binop, PyDict_SetItem(d, key, binop(init, item)) -cpdef dict reduceby(object key, object binop, object seq, object init=no_default): +cpdef dict reduceby(object key, object binop, object seq, object init='__no__default__'): """ Perform a simultaneous groupby and reduction @@ -952,7 +952,7 @@ cdef class sliding_window: no_pad = '__no__pad__' -cpdef object partition(Py_ssize_t n, object seq, object pad=no_pad): +cpdef object partition(Py_ssize_t n, object seq, object pad='__no__pad__'): """ Partition sequence into tuples of length n @@ -972,7 +972,7 @@ cpdef object partition(Py_ssize_t n, object seq, object pad=no_pad): partition_all """ args = [iter(seq)] * n - if pad is no_pad: + if pad == '__no__pad__': return zip(*args) else: return zip_longest(*args, fillvalue=pad) @@ -1127,7 +1127,7 @@ cdef class _pluck_list_default: return result -cpdef object pluck(object ind, object seqs, object default=no_default): +cpdef object pluck(object ind, object seqs, object default='__no__default__'): """ plucks an element or several elements from each item in a sequence. @@ -1206,8 +1206,8 @@ cpdef object getter(object index): cpdef object join(object leftkey, object leftseq, object rightkey, object rightseq, - object left_default=no_default, - object right_default=no_default): + object left_default='__no__default__', + object right_default='__no__default__'): """ Join two sequences on common attributes diff --git a/cytoolz/tests/test_embedded_sigs.py b/cytoolz/tests/test_embedded_sigs.py index 2046697..b360c91 100644 --- a/cytoolz/tests/test_embedded_sigs.py +++ b/cytoolz/tests/test_embedded_sigs.py @@ -9,7 +9,7 @@ @curry def isfrommod(modname, func): mod = getattr(func, '__module__', '') or '' - return modname in mod + return mod.startswith(modname) or 'toolz.functoolz.curry' in str(type(func)) @dev_skip_test @@ -45,8 +45,16 @@ def test_class_sigs(): # class toolz_spec = inspect.getargspec(toolz_func.__init__) + # For Cython < 0.25 toolz_sig = toolz_func.__name__ + inspect.formatargspec(*toolz_spec) - if toolz_sig not in cytoolz_func.__doc__: + doc = cytoolz_func.__doc__ + # For Cython >= 0.25 + toolz_sig_alt = toolz_func.__name__ + inspect.formatargspec( + *toolz_spec, + **{'formatvalue': lambda x: '=' + getattr(x, '__name__', repr(x))} + ) + doc_alt = doc.replace('Py_ssize_t ', '') + if not (toolz_sig in doc or toolz_sig_alt in doc_alt): message = ('cytoolz.%s does not have correct function signature.' '\n\nExpected: %s' '\n\nDocstring in cytoolz is:\n%s' diff --git a/cytoolz/tests/test_none_safe.py b/cytoolz/tests/test_none_safe.py index cf6c7e8..8dbd7cc 100644 --- a/cytoolz/tests/test_none_safe.py +++ b/cytoolz/tests/test_none_safe.py @@ -282,7 +282,7 @@ def test_itertoolz(): tested.append('take_nth') assert raises(TypeError, lambda: list(unique(None))) - assert raises(TypeError, lambda: list(unique([1, 1, 2], key=None))) + assert list(unique([1, 1, 2], key=None)) == [1, 2] tested.append('unique') assert raises(TypeError, lambda: join(first, None, second, (1, 2, 3))) From 52711fe6ca1386f6ae149133ee91bf7e23280b8f Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 1 Nov 2016 10:06:19 -0500 Subject: [PATCH 106/146] Faster `merge_sorted` using binary merge. This is 2-3x faster than the fast `toolz` version, and 4-5x faster than the previous `cytoolz` version. Hooray! --- cytoolz/itertoolz.pxd | 17 +++- cytoolz/itertoolz.pyx | 208 ++++++++++++++++++++++-------------------- 2 files changed, 120 insertions(+), 105 deletions(-) diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index 546fc83..55f798e 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -14,14 +14,21 @@ cpdef dict groupby(object key, object seq) cdef class _merge_sorted: - cdef list pq - cdef object shortcut - + cdef object seq1 + cdef object seq2 + cdef object val1 + cdef object val2 + cdef Py_ssize_t loop cdef class _merge_sorted_key: - cdef list pq + cdef object seq1 + cdef object seq2 + cdef object val1 + cdef object val2 cdef object key - cdef object shortcut + cdef object key1 + cdef object key2 + cdef Py_ssize_t loop cdef object c_merge_sorted(object seqs, object key=*) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 639bcd0..54f2792 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -167,123 +167,131 @@ cpdef dict groupby(object key, object seq): return d -cdef class _merge_sorted: - def __cinit__(self, seqs): - cdef Py_ssize_t i - cdef object item, it - self.pq = [] - self.shortcut = None +cdef object _merge_sorted_binary(object seqs): + mid = len(seqs) // 2 + L1 = seqs[:mid] + if len(L1) == 1: + seq1 = iter(L1[0]) + else: + seq1 = _merge_sorted_binary(L1) + L2 = seqs[mid:] + if len(L2) == 1: + seq2 = iter(L2[0]) + else: + seq2 = _merge_sorted_binary(L2) + try: + val2 = next(seq2) + except StopIteration: + return seq1 + return _merge_sorted(seq1, seq2, val2) - for i, item in enumerate(seqs): - it = iter(item) - try: - item = next(it) - PyList_Append(self.pq, [item, i, it]) - except StopIteration: - pass - i = PyList_GET_SIZE(self.pq) - if i == 0: - self.shortcut = iter([]) - elif i == 1: - self.shortcut = True - else: - heapify(self.pq) + +cdef class _merge_sorted: + def __cinit__(self, seq1, seq2, val2): + self.seq1 = seq1 + self.seq2 = seq2 + self.val1 = None + self.val2 = val2 + self.loop = 0 def __iter__(self): return self def __next__(self): - cdef list item - cdef object retval, it - # Fast when only a single iterator remains - if self.shortcut is not None: - if self.shortcut is True: - item = self.pq[0] - self.shortcut = item[2] - return item[0] - return next(self.shortcut) - - item = self.pq[0] - retval = item[0] - it = item[2] - try: - item[0] = next(it) - heapreplace(self.pq, item) - except StopIteration: - heappop(self.pq) - if PyList_GET_SIZE(self.pq) == 1: - self.shortcut = True - return retval - - -# Having `_merge_sorted` and `_merge_sorted_key` separate violates the DRY -# principle. The increased performance *barely* justifies this. -# `_merge_sorted` is always faster (sometimes by only 15%), but it can be -# more than 3x faster when a single iterable remains. -# -# The differences in implementation are that `_merge_sorted_key` calls a key -# function on each item (of course), and the layout of the lists in the -# priority queue are different: -# `_merge_sorted` uses `[item, itnum, iterator]` -# `_merge_sorted_key` uses `[key(item), itnum, item, iterator]` + if self.loop == 0: + try: + self.val1 = next(self.seq1) + except StopIteration: + self.loop = 2 + return self.val2 + if self.val2 < self.val1: + self.loop = 1 + return self.val2 + return self.val1 + elif self.loop == 1: + try: + self.val2 = next(self.seq2) + except StopIteration: + self.loop = 3 + return self.val1 + if self.val2 < self.val1: + return self.val2 + self.loop = 0 + return self.val1 + elif self.loop == 2: + return next(self.seq2) + return next(self.seq1) + + +cdef object _merge_sorted_binary_key(object seqs, object key): + mid = len(seqs) // 2 + L1 = seqs[:mid] + if len(L1) == 1: + seq1 = iter(L1[0]) + else: + seq1 = _merge_sorted_binary_key(L1, key) + L2 = seqs[mid:] + if len(L2) == 1: + seq2 = iter(L2[0]) + else: + seq2 = _merge_sorted_binary_key(L2, key) + try: + val2 = next(seq2) + except StopIteration: + return seq1 + return _merge_sorted_key(seq1, seq2, val2, key) + cdef class _merge_sorted_key: - def __cinit__(self, seqs, key): - cdef Py_ssize_t i - cdef object item, it, k - self.pq = [] + def __cinit__(self, seq1, seq2, val2, key): + self.seq1 = seq1 + self.seq2 = seq2 self.key = key - self.shortcut = None - - for i, item in enumerate(seqs): - it = iter(item) - try: - item = next(it) - k = key(item) - PyList_Append(self.pq, [k, i, item, it]) - except StopIteration: - pass - i = PyList_GET_SIZE(self.pq) - if i == 0: - self.shortcut = iter([]) - elif i == 1: - self.shortcut = True - else: - heapify(self.pq) + self.val1 = None + self.key1 = None + self.val2 = val2 + self.key2 = key(val2) + self.loop = 0 def __iter__(self): return self def __next__(self): - cdef list item - cdef object retval, it, k - # Fast when only a single iterator remains - if self.shortcut is not None: - if self.shortcut is True: - item = self.pq[0] - self.shortcut = item[3] - return item[2] - return next(self.shortcut) - - item = self.pq[0] - retval = item[2] - it = item[3] - try: - k = next(it) - item[2] = k - item[0] = self.key(k) - heapreplace(self.pq, item) - except StopIteration: - heappop(self.pq) - if PyList_GET_SIZE(self.pq) == 1: - self.shortcut = True - return retval + if self.loop == 0: + try: + self.val1 = next(self.seq1) + except StopIteration: + self.loop = 2 + return self.val2 + self.key1 = self.key(self.val1) + if self.key2 < self.key1: + self.loop = 1 + return self.val2 + return self.val1 + elif self.loop == 1: + try: + self.val2 = next(self.seq2) + except StopIteration: + self.loop = 3 + return self.val1 + self.key2 = self.key(self.val2) + if self.key2 < self.key1: + return self.val2 + self.loop = 0 + return self.val1 + elif self.loop == 2: + return next(self.seq2) + return next(self.seq1) cdef object c_merge_sorted(object seqs, object key=None): - if key is None: - return _merge_sorted(seqs) - return _merge_sorted_key(seqs, key) + if len(seqs) == 0: + return iter([]) + elif len(seqs) == 1: + return iter(seqs[0]) + elif key is None: + return _merge_sorted_binary(seqs) + return _merge_sorted_binary_key(seqs, key) def merge_sorted(*seqs, **kwargs): From 0f54c825c3a4e0215ca63d94db31616fff3e5058 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 3 Nov 2016 00:46:52 -0500 Subject: [PATCH 107/146] Update `cytoolz` to the latest dev version of `toolz`. 1. `memoize` and `flip` are curried now, and `c_memoize` and `c_flip` were removed. 2. Update doctest of `flip`. 3. Improved pickling of curried objects in top-level modules. 4. Added `_should_curry` (even though it's an implementation detail atm) to `curry`. 5. `pass_exceptions=` keyword removed from `interleave`. 6. Updated tests and `toolz.curried` via make commands. --- cytoolz/__init__.pxd | 4 +- cytoolz/__init__.py | 4 ++ cytoolz/_signatures.py | 12 +++--- cytoolz/compatibility.py | 9 +++- cytoolz/curried/__init__.py | 1 - cytoolz/functoolz.pxd | 4 +- cytoolz/functoolz.pyx | 64 +++++++++++++++++++---------- cytoolz/itertoolz.pxd | 1 - cytoolz/itertoolz.pyx | 11 ++--- cytoolz/tests/test_docstrings.py | 2 +- cytoolz/tests/test_itertoolz.py | 7 ++++ cytoolz/tests/test_serialization.py | 15 +++++++ 12 files changed, 92 insertions(+), 42 deletions(-) diff --git a/cytoolz/__init__.pxd b/cytoolz/__init__.pxd index 88cd4bf..24c7b02 100644 --- a/cytoolz/__init__.pxd +++ b/cytoolz/__init__.pxd @@ -7,8 +7,8 @@ from cytoolz.itertoolz cimport ( from cytoolz.functoolz cimport ( - c_compose, c_juxt, c_memoize, c_pipe, c_thread_first, c_thread_last, - complement, curry, do, identity, excepts, c_flip) + c_compose, c_juxt, memoize, c_pipe, c_thread_first, c_thread_last, + complement, curry, do, identity, excepts, flip) from cytoolz.dicttoolz cimport ( diff --git a/cytoolz/__init__.py b/cytoolz/__init__.py index 8c2eedd..01746e9 100644 --- a/cytoolz/__init__.py +++ b/cytoolz/__init__.py @@ -17,6 +17,10 @@ # Aliases comp = compose +# Always-curried functions +flip = functoolz.flip = curry(functoolz.flip) +memoize = functoolz.memoize = curry(functoolz.memoize) + functoolz._sigs.update_signature_registry() from ._version import __version__, __toolz_version__ diff --git a/cytoolz/_signatures.py b/cytoolz/_signatures.py index d7fb24e..9ee6360 100644 --- a/cytoolz/_signatures.py +++ b/cytoolz/_signatures.py @@ -36,10 +36,6 @@ cytoolz_info['cytoolz.functoolz'] = dict( Compose=[ lambda *funcs: None], - c_flip=[ - lambda func, a, b: None], - c_memoize=[ - lambda func, cache=None, key=None: None], complement=[ lambda func: None], compose=[ @@ -50,12 +46,14 @@ lambda func, x: None], excepts=[ lambda exc, func, handler=None: None], + flip=[ # XXX: these are optional, but not keywords! + lambda func=None, a=None, b=None: None], identity=[ lambda x: None], juxt=[ lambda *funcs: None], - memoize=[ - lambda func, cache=None, key=None: None], + memoize=[ # XXX: func is optional, but not a keyword! + lambda func=None, cache=None, key=None: None], pipe=[ lambda data, *funcs: None], return_none=[ @@ -90,7 +88,7 @@ identity=[ lambda x: None], interleave=[ - lambda seqs, pass_exceptions=(): None], + lambda seqs: None], interpose=[ lambda el, seq: None], isdistinct=[ diff --git a/cytoolz/compatibility.py b/cytoolz/compatibility.py index fef5acc..595d43e 100644 --- a/cytoolz/compatibility.py +++ b/cytoolz/compatibility.py @@ -5,7 +5,7 @@ PY34 = sys.version_info[0] == 3 and sys.version_info[1] == 4 __all__ = ['PY3', 'map', 'filter', 'range', 'zip', 'reduce', 'zip_longest', - 'iteritems', 'iterkeys', 'itervalues'] + 'iteritems', 'iterkeys', 'itervalues', 'import_module'] if PY3: map = map @@ -27,3 +27,10 @@ iteritems = operator.methodcaller('iteritems') iterkeys = operator.methodcaller('iterkeys') itervalues = operator.methodcaller('itervalues') + +try: + from importlib import import_module +except ImportError: + def import_module(name): + __import__(name) + return sys.modules[name] diff --git a/cytoolz/curried/__init__.py b/cytoolz/curried/__init__.py index 174c9a0..474ec92 100644 --- a/cytoolz/curried/__init__.py +++ b/cytoolz/curried/__init__.py @@ -48,7 +48,6 @@ cytoolz.get, cytoolz.get_in, cytoolz.groupby, - cytoolz.interleave, cytoolz.interpose, cytoolz.itemfilter, cytoolz.itemmap, diff --git a/cytoolz/functoolz.pxd b/cytoolz/functoolz.pxd index 7c7a515..0eee035 100644 --- a/cytoolz/functoolz.pxd +++ b/cytoolz/functoolz.pxd @@ -16,7 +16,7 @@ cdef class curry: cdef public object __doc__ cdef public object __name__ -cdef class c_memoize: +cdef class memoize: cdef object func cdef object cache cdef object key @@ -49,7 +49,7 @@ cdef object c_juxt(object funcs) cpdef object do(object func, object x) -cpdef object c_flip(object func, object a, object b) +cpdef object flip(object func, object a, object b) cpdef object return_none(object exc) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index d86adbf..4495a4a 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -4,7 +4,7 @@ from functools import partial from operator import attrgetter from textwrap import dedent from cytoolz.utils import no_default -from cytoolz.compatibility import PY3, PY34, filter as ifilter, map as imap, reduce +from cytoolz.compatibility import PY3, PY34, filter as ifilter, map as imap, reduce, import_module import cytoolz._signatures as _sigs from toolz.functoolz import (InstanceProperty, instanceproperty, is_arity, @@ -233,6 +233,15 @@ cdef class curry: return type(self)(self.func, *args, **kwargs) raise + def _should_curry(self, args, kwargs, exc=None): + if PyTuple_GET_SIZE(args) == 0: + args = self.args + elif PyTuple_GET_SIZE(self.args) != 0: + args = PySequence_Concat(self.args, args) + if self.keywords is not None: + PyDict_Merge(kwargs, self.keywords, False) + return self._should_curry_internal(args, kwargs) + def _should_curry_internal(self, args, kwargs, exc=None): func = self.func @@ -281,12 +290,6 @@ cdef class curry: return self return type(self)(self, instance) - def __reduce__(self): - return (type(self), (self.func,), (self.args, self.keywords)) - - def __setstate__(self, state): - self.args, self.keywords = state - property __signature__: def __get__(self): sig = inspect.signature(self.func) @@ -325,8 +328,33 @@ cdef class curry: return sig.replace(parameters=newparams) + def __reduce__(self): + func = self.func + modname = getattr(func, '__module__', None) + funcname = getattr(func, '__name__', None) + if modname and funcname: + module = import_module(modname) + obj = getattr(module, funcname, None) + if obj is self: + return funcname + elif isinstance(obj, curry) and obj.func is func: + func = '%s.%s' % (modname, funcname) -cdef class c_memoize: + state = (type(self), func, self.args, self.keywords) + return (_restore_curry, state) + + +cpdef object _restore_curry(cls, func, args, kwargs): + if isinstance(func, str): + modname, funcname = func.rsplit('.', 1) + module = import_module(modname) + func = getattr(module, funcname).func + obj = cls(func, *args, **(kwargs or {})) + return obj + + + +cdef class memoize: """ memoize(func, cache=None, key=None) Cache a function's result for speedy future evaluation @@ -418,9 +446,6 @@ cdef class c_memoize: return curry(self, instance) -memoize = curry(c_memoize) - - cdef class Compose: """ Compose(self, *funcs) @@ -628,22 +653,20 @@ cpdef object do(object func, object x): return x -@cython.embedsignature(False) -cpdef object c_flip(object func, object a, object b): - """ flip(func, a, b) - +cpdef object flip(object func, object a, object b): + """ Call the function call with the arguments flipped This function is curried. >>> def div(a, b): - ... return a / b + ... return a // b ... - >>> flip(div, 2, 1) - 0.5 + >>> flip(div, 2, 6) + 3 >>> div_by_two = flip(div, 2) >>> div_by_two(4) - 2.0 + 2 This is particularly useful for built in functions and functions defined in C extensions that accept positional only arguments. For example: @@ -657,9 +680,6 @@ cpdef object c_flip(object func, object a, object b): return PyObject_CallObject(func, (b, a)) -flip = curry(c_flip) - - cpdef object return_none(object exc): """ Returns None. diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index 55f798e..75d4578 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -37,7 +37,6 @@ cdef object c_merge_sorted(object seqs, object key=*) cdef class interleave: cdef list iters cdef list newiters - cdef tuple pass_exceptions cdef Py_ssize_t i cdef Py_ssize_t n diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index d2bf149..a25ad8d 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -317,7 +317,7 @@ def merge_sorted(*seqs, **kwargs): cdef class interleave: - """ interleave(seqs, pass_exceptions=()) + """ interleave(seqs) Interleave a sequence of sequences @@ -331,10 +331,9 @@ cdef class interleave: Returns a lazy iterator """ - def __cinit__(self, seqs, pass_exceptions=()): + def __cinit__(self, seqs): self.iters = [iter(seq) for seq in seqs] self.newiters = [] - self.pass_exceptions = tuple(pass_exceptions) self.i = 0 self.n = PyList_GET_SIZE(self.iters) @@ -359,13 +358,15 @@ cdef class interleave: self.i += 1 obj = PtrIter_Next(val) + # TODO: optimization opportunity. Previously, it was possible to + # continue on given exceptions, `self.pass_exceptions`, which is + # why this code is structured this way. Time to clean up? while obj is NULL: obj = PyErr_Occurred() if obj is not NULL: val = obj PyErr_Clear() - if not PyErr_GivenExceptionMatches(val, self.pass_exceptions): - raise val + raise val if self.i == self.n: self.n = PyList_GET_SIZE(self.newiters) diff --git a/cytoolz/tests/test_docstrings.py b/cytoolz/tests/test_docstrings.py index d9711b1..85654e6 100644 --- a/cytoolz/tests/test_docstrings.py +++ b/cytoolz/tests/test_docstrings.py @@ -15,7 +15,7 @@ @curry def isfrommod(modname, func): mod = getattr(func, '__module__', '') or '' - return modname in mod + return mod.startswith(modname) or 'toolz.functoolz.curry' in str(type(func)) def convertdoc(doc): diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index 9f35716..c7d99b8 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -86,6 +86,13 @@ def test_merge_sorted(): [(9, 1), (9, 8), (9, 9)]] assert list(merge_sorted(*data, key=lambda x: x[1])) == [ (9, 1), (1, 2), (5, 3), (0, 4), (6, 5), (3, 6), (8, 8), (9, 8), (9, 9)] + assert list(merge_sorted()) == [] + assert list(merge_sorted([1, 2, 3])) == [1, 2, 3] + assert list(merge_sorted([1, 4, 5], [2, 3])) == [1, 2, 3, 4, 5] + assert list(merge_sorted([1, 4, 5], [2, 3], key=identity)) == [ + 1, 2, 3, 4, 5] + assert list(merge_sorted([1, 5], [2], [4, 7], [3, 6], key=identity)) == [ + 1, 2, 3, 4, 5, 6, 7] def test_interleave(): diff --git a/cytoolz/tests/test_serialization.py b/cytoolz/tests/test_serialization.py index 39686f6..f5522b4 100644 --- a/cytoolz/tests/test_serialization.py +++ b/cytoolz/tests/test_serialization.py @@ -40,3 +40,18 @@ def test_instanceproperty(): assert p2.__get__(None) is None assert p2.__get__(0) is False assert p2.__get__(1) is True + + +def f(x, y): + return x, y + + +def test_flip(): + flip = pickle.loads(pickle.dumps(cytoolz.functoolz.flip)) + assert flip is cytoolz.functoolz.flip + g1 = flip(f) + g2 = pickle.loads(pickle.dumps(g1)) + assert g1(1, 2) == g2(1, 2) == f(2, 1) + g1 = flip(f)(1) + g2 = pickle.loads(pickle.dumps(g1)) + assert g1(2) == g2(2) == f(2, 1) From 301c1c26fcb6d62bb99da9dc02de3d3c6450eb10 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 3 Nov 2016 01:23:52 -0500 Subject: [PATCH 108/146] Maybe fix introspection of curried cytoolz objects. --- cytoolz/_signatures.py | 4 ++++ cytoolz/functoolz.pyx | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/cytoolz/_signatures.py b/cytoolz/_signatures.py index 9ee6360..733afe5 100644 --- a/cytoolz/_signatures.py +++ b/cytoolz/_signatures.py @@ -48,12 +48,16 @@ lambda exc, func, handler=None: None], flip=[ # XXX: these are optional, but not keywords! lambda func=None, a=None, b=None: None], + _flip=[ + lambda func, a, b: None], identity=[ lambda x: None], juxt=[ lambda *funcs: None], memoize=[ # XXX: func is optional, but not a keyword! lambda func=None, cache=None, key=None: None], + _memoize=[ + lambda func, cache=None, key=None: None], pipe=[ lambda data, *funcs: None], return_none=[ diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 4495a4a..54a2ba4 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -446,6 +446,9 @@ cdef class memoize: return curry(self, instance) +_memoize = memoize # uncurried + + cdef class Compose: """ Compose(self, *funcs) @@ -680,6 +683,9 @@ cpdef object flip(object func, object a, object b): return PyObject_CallObject(func, (b, a)) +_flip = flip # uncurried + + cpdef object return_none(object exc): """ Returns None. From 2195a947064d7c5819acaac974d21e7e1af0578d Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 3 Nov 2016 09:51:47 -0500 Subject: [PATCH 109/146] Update minimum required Cython version from 0.17 to 0.20.2 --- requirements_devel.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_devel.txt b/requirements_devel.txt index c6a899d..9c8944a 100644 --- a/requirements_devel.txt +++ b/requirements_devel.txt @@ -1,3 +1,3 @@ -cython>=0.17 +cython>=0.20.2 nose -e git+https://github.com/pytoolz/toolz.git From 724eeb164c1ad00691471281dc8465d0c1e6d2da Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 3 Nov 2016 13:52:13 -0500 Subject: [PATCH 110/146] More changes to match recent changes in `toolz`. --- Makefile | 2 ++ cytoolz/__init__.pxd | 2 +- cytoolz/_signatures.py | 4 +++ cytoolz/curried/exceptions.py | 1 - cytoolz/itertoolz.pxd | 3 +++ cytoolz/itertoolz.pyx | 39 +++++++++++++++++++++++----- cytoolz/tests/test_curried.py | 49 +++++++++++++++++++++++++++++++++++ 7 files changed, 92 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index ae22ed3..0a67858 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,8 @@ copytests: for f in ../toolz/toolz/tests/test*py; \ do \ if [[ $$f == *test_utils* ]]; then continue ; fi; \ + if [[ $$f == *test_curried_doctests* ]]; then continue ; fi; \ + if [[ $$f == *test_tlz* ]]; then continue ; fi; \ newf=`echo $$f | sed 's/...toolz.toolz/cytoolz/g'`; \ sed -e 's/toolz/cytoolz/g' -e 's/itercytoolz/itertoolz/' \ -e 's/dictcytoolz/dicttoolz/g' -e 's/funccytoolz/functoolz/g' \ diff --git a/cytoolz/__init__.pxd b/cytoolz/__init__.pxd index 24c7b02..471f464 100644 --- a/cytoolz/__init__.pxd +++ b/cytoolz/__init__.pxd @@ -3,7 +3,7 @@ from cytoolz.itertoolz cimport ( frequencies, interleave, interpose, isdistinct, isiterable, iterate, last, mapcat, nth, partition, partition_all, pluck, reduceby, remove, rest, second, sliding_window, take, tail, take_nth, unique, join, - c_diff, topk, peek, random_sample) + c_diff, topk, peek, random_sample, concat) from cytoolz.functoolz cimport ( diff --git a/cytoolz/_signatures.py b/cytoolz/_signatures.py index 733afe5..cea1de6 100644 --- a/cytoolz/_signatures.py +++ b/cytoolz/_signatures.py @@ -71,6 +71,10 @@ cytoolz_info['cytoolz.itertoolz'] = dict( accumulate=[ lambda binop, seq, initial='__no__default__': None], + concat=[ + lambda seqs: None], + concatv=[ + lambda *seqs: None], cons=[ lambda el, seq: None], count=[ diff --git a/cytoolz/curried/exceptions.py b/cytoolz/curried/exceptions.py index ce09b3d..70dd8bc 100644 --- a/cytoolz/curried/exceptions.py +++ b/cytoolz/curried/exceptions.py @@ -1,5 +1,4 @@ import cytoolz -#from cytoolz import curry, merge as _merge, merge_with as _merge_with __all__ = ['merge', 'merge_with'] diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index 75d4578..9684e44 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -94,6 +94,9 @@ cpdef object get(object ind, object seq, object default=*) cpdef object cons(object el, object seq) +cpdef object concat(object seqs) + + cpdef object mapcat(object func, object seqs) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index a25ad8d..262619c 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -27,10 +27,6 @@ __all__ = ['remove', 'accumulate', 'groupby', 'merge_sorted', 'interleave', 'join', 'tail', 'diff', 'topk', 'peek', 'random_sample'] -concatv = chain -concat = chain.from_iterable - - cpdef object identity(object x): return x @@ -695,6 +691,38 @@ cpdef object get(object ind, object seq, object default='__no__default__'): return obj +cpdef object concat(object seqs): + """ + Concatenate zero or more iterables, any of which may be infinite. + + An infinite sequence will prevent the rest of the arguments from + being included. + + We use chain.from_iterable rather than ``chain(*seqs)`` so that seqs + can be a generator. + + >>> list(concat([[], [1], [2, 3]])) + [1, 2, 3] + + See also: + itertools.chain.from_iterable equivalent + """ + return chain.from_iterable(seqs) + + +def concatv(*seqs): + """ + Variadic version of concat + + >>> list(concatv([], ["a"], ["b", "c"])) + ['a', 'b', 'c'] + + See also: + itertools.chain + """ + return chain.from_iterable(seqs) + + cpdef object mapcat(object func, object seqs): """ Apply func to each sequence in seqs, concatenating results. @@ -1145,8 +1173,7 @@ cpdef object pluck(object ind, object seqs, object default='__no__default__'): This is equivalent to running `map(curried.get(ind), seqs)` - ``ind`` can be either a single string/index or a sequence of - strings/indices. + ``ind`` can be either a single string/index or a list of strings/indices. ``seqs`` should be sequence containing sequences or dicts. e.g. diff --git a/cytoolz/tests/test_curried.py b/cytoolz/tests/test_curried.py index da15ff8..a2ef1ad 100644 --- a/cytoolz/tests/test_curried.py +++ b/cytoolz/tests/test_curried.py @@ -2,6 +2,7 @@ import cytoolz.curried from cytoolz.curried import (take, first, second, sorted, merge_with, reduce, merge, operator as cop) +from cytoolz.compatibility import import_module from collections import defaultdict from operator import add @@ -62,3 +63,51 @@ def test_curried_operator(): # Make sure this isn't totally empty. assert len(set(vars(cop)) & set(['add', 'sub', 'mul'])) == 3 + + +def test_curried_namespace(): + exceptions = import_module('cytoolz.curried.exceptions') + namespace = {} + + def should_curry(func): + if not callable(func) or isinstance(func, cytoolz.curry): + return False + nargs = cytoolz.functoolz.num_required_args(func) + if nargs is None or nargs > 1: + return True + return nargs == 1 and cytoolz.functoolz.has_keywords(func) + + + def curry_namespace(ns): + return dict( + (name, cytoolz.curry(f) if should_curry(f) else f) + for name, f in ns.items() if '__' not in name + ) + + from_cytoolz = curry_namespace(vars(cytoolz)) + from_exceptions = curry_namespace(vars(exceptions)) + namespace.update(cytoolz.merge(from_cytoolz, from_exceptions)) + + namespace = cytoolz.valfilter(callable, namespace) + curried_namespace = cytoolz.valfilter(callable, cytoolz.curried.__dict__) + + if namespace != curried_namespace: + missing = set(namespace) - set(curried_namespace) + if missing: + raise AssertionError('There are missing functions in cytoolz.curried:\n %s' + % ' \n'.join(sorted(missing))) + extra = set(curried_namespace) - set(namespace) + if extra: + raise AssertionError('There are extra functions in cytoolz.curried:\n %s' + % ' \n'.join(sorted(extra))) + unequal = cytoolz.merge_with(list, namespace, curried_namespace) + unequal = cytoolz.valfilter(lambda x: x[0] != x[1], unequal) + messages = [] + for name, (orig_func, auto_func) in sorted(unequal.items()): + if name in from_exceptions: + messages.append('%s should come from cytoolz.curried.exceptions' % name) + elif should_curry(getattr(cytoolz, name)): + messages.append('%s should be curried from cytoolz' % name) + else: + messages.append('%s should come from cytoolz and NOT be curried' % name) + raise AssertionError('\n'.join(messages)) From 58f3b84e53a8d69fb344c964ecff2fe604b4e432 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 3 Nov 2016 22:54:08 -0500 Subject: [PATCH 111/146] Explicitly curry objects in cytoolz.curried by copying what toolz does. As before run `make curried`, which executes a `sed` command. "etc/generate_curried.py" is no longer required. We still have tests that verify whether functions should or shouldn't be curried, and whether cytoolz matches toolz. Also, add tests for `tlz` and assume `cytoolz` is installed. --- Makefile | 4 +- cytoolz/curried/__init__.py | 145 ++++++++++++++++++------------------ cytoolz/tests/test_tlz.py | 54 ++++++++++++++ etc/generate_curried.py | 98 ------------------------ 4 files changed, 128 insertions(+), 173 deletions(-) create mode 100644 cytoolz/tests/test_tlz.py delete mode 100755 etc/generate_curried.py diff --git a/Makefile b/Makefile index 0a67858..e01bc95 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,9 @@ clean: rm -rf build/ cytoolz/__pycache__/ cytoolz/*/__pycache__/ curried: - $(PYTHON) etc/generate_curried.py > cytoolz/curried/__init__.py + sed -e 's/toolz/cytoolz/g' -e 's/itercytoolz/itertoolz/' \ + -e 's/dictcytoolz/dicttoolz/g' -e 's/funccytoolz/functoolz/g' \ + ../toolz/toolz/curried/__init__.py > cytoolz/curried/__init__.py copytests: for f in ../toolz/toolz/tests/test*py; \ diff --git a/cytoolz/curried/__init__.py b/cytoolz/curried/__init__.py index 474ec92..a27783d 100644 --- a/cytoolz/curried/__init__.py +++ b/cytoolz/curried/__init__.py @@ -1,11 +1,5 @@ -############################################################################### -# THIS FILE IS AUTOMATICALLY GENERATED # -# # -# Please make all edits in etc/generate_curried.py and re-run # -# cytoolz/tests/test_curried_toolzlike.py # -############################################################################### """ -Alternate namespece for cytoolz such that all functions are curried +Alternate namespace for cytoolz such that all functions are curried Currying provides implicit partial evaluation of all functions @@ -29,75 +23,78 @@ See Also: cytoolz.functoolz.curry """ -from . import exceptions -from . import operator import cytoolz +from . import operator +from cytoolz import ( + comp, + complement, + compose, + concat, + concatv, + count, + curry, + diff, + dissoc, + first, + flip, + frequencies, + identity, + interleave, + isdistinct, + isiterable, + juxt, + last, + memoize, + merge_sorted, + peek, + pipe, + second, + thread_first, + thread_last, +) +from .exceptions import merge, merge_with +accumulate = cytoolz.curry(cytoolz.accumulate) +assoc = cytoolz.curry(cytoolz.assoc) +assoc_in = cytoolz.curry(cytoolz.assoc_in) +cons = cytoolz.curry(cytoolz.cons) +countby = cytoolz.curry(cytoolz.countby) +do = cytoolz.curry(cytoolz.do) +drop = cytoolz.curry(cytoolz.drop) +excepts = cytoolz.curry(cytoolz.excepts) +filter = cytoolz.curry(cytoolz.filter) +get = cytoolz.curry(cytoolz.get) +get_in = cytoolz.curry(cytoolz.get_in) +groupby = cytoolz.curry(cytoolz.groupby) +interpose = cytoolz.curry(cytoolz.interpose) +itemfilter = cytoolz.curry(cytoolz.itemfilter) +itemmap = cytoolz.curry(cytoolz.itemmap) +iterate = cytoolz.curry(cytoolz.iterate) +join = cytoolz.curry(cytoolz.join) +keyfilter = cytoolz.curry(cytoolz.keyfilter) +keymap = cytoolz.curry(cytoolz.keymap) +map = cytoolz.curry(cytoolz.map) +mapcat = cytoolz.curry(cytoolz.mapcat) +nth = cytoolz.curry(cytoolz.nth) +partial = cytoolz.curry(cytoolz.partial) +partition = cytoolz.curry(cytoolz.partition) +partition_all = cytoolz.curry(cytoolz.partition_all) +partitionby = cytoolz.curry(cytoolz.partitionby) +pluck = cytoolz.curry(cytoolz.pluck) +random_sample = cytoolz.curry(cytoolz.random_sample) +reduce = cytoolz.curry(cytoolz.reduce) +reduceby = cytoolz.curry(cytoolz.reduceby) +remove = cytoolz.curry(cytoolz.remove) +sliding_window = cytoolz.curry(cytoolz.sliding_window) +sorted = cytoolz.curry(cytoolz.sorted) +tail = cytoolz.curry(cytoolz.tail) +take = cytoolz.curry(cytoolz.take) +take_nth = cytoolz.curry(cytoolz.take_nth) +topk = cytoolz.curry(cytoolz.topk) +unique = cytoolz.curry(cytoolz.unique) +update_in = cytoolz.curry(cytoolz.update_in) +valfilter = cytoolz.curry(cytoolz.valfilter) +valmap = cytoolz.curry(cytoolz.valmap) -_curry_set = frozenset([ - cytoolz.accumulate, - cytoolz.assoc, - cytoolz.assoc_in, - cytoolz.cons, - cytoolz.countby, - cytoolz.do, - cytoolz.drop, - cytoolz.excepts, - cytoolz.filter, - cytoolz.flip, - cytoolz.get, - cytoolz.get_in, - cytoolz.groupby, - cytoolz.interpose, - cytoolz.itemfilter, - cytoolz.itemmap, - cytoolz.iterate, - cytoolz.join, - cytoolz.keyfilter, - cytoolz.keymap, - cytoolz.map, - cytoolz.mapcat, - cytoolz.memoize, - cytoolz.merge, - cytoolz.merge_with, - cytoolz.nth, - cytoolz.partial, - cytoolz.partition, - cytoolz.partition_all, - cytoolz.partitionby, - cytoolz.pluck, - cytoolz.random_sample, - cytoolz.reduce, - cytoolz.reduceby, - cytoolz.remove, - cytoolz.sliding_window, - cytoolz.sorted, - cytoolz.tail, - cytoolz.take, - cytoolz.take_nth, - cytoolz.topk, - cytoolz.unique, - cytoolz.update_in, - cytoolz.valfilter, - cytoolz.valmap, -]) - - -def _curry_namespace(ns): - return dict( - (name, cytoolz.curry(f) if f in _curry_set else f) - for name, f in ns.items() if '__' not in name - ) - - -locals().update(cytoolz.merge( - _curry_namespace(vars(cytoolz)), - _curry_namespace(vars(exceptions)), -)) - -# Clean up the namespace. -del _curry_set -del _curry_namespace del exceptions del cytoolz - diff --git a/cytoolz/tests/test_tlz.py b/cytoolz/tests/test_tlz.py new file mode 100644 index 0000000..d1fe791 --- /dev/null +++ b/cytoolz/tests/test_tlz.py @@ -0,0 +1,54 @@ +import toolz + + +def test_tlz(): + import tlz + tlz.curry + tlz.functoolz.curry + assert tlz.__package__ == 'tlz' + assert tlz.__name__ == 'tlz' + import tlz.curried + assert tlz.curried.__package__ == 'tlz.curried' + assert tlz.curried.__name__ == 'tlz.curried' + tlz.curried.curry + import tlz.curried.operator + assert tlz.curried.operator.__package__ in (None, 'tlz.curried') + assert tlz.curried.operator.__name__ == 'tlz.curried.operator' + assert tlz.functoolz.__name__ == 'tlz.functoolz' + m1 = tlz.functoolz + import tlz.functoolz as m2 + assert m1 is m2 + import tlz.sandbox + try: + import tlzthisisabadname.curried + 1/0 + except ImportError: + pass + try: + import tlz.curry + 1/0 + except ImportError: + pass + try: + import tlz.badsubmodulename + 1/0 + except ImportError: + pass + + assert toolz.__package__ == 'toolz' + assert toolz.curried.__package__ == 'toolz.curried' + assert toolz.functoolz.__name__ == 'toolz.functoolz' + + import cytoolz + assert cytoolz.__package__ == 'cytoolz' + assert cytoolz.curried.__package__ == 'cytoolz.curried' + assert cytoolz.functoolz.__name__ == 'cytoolz.functoolz' + assert tlz.pluck is cytoolz.pluck + + assert tlz.__file__ == toolz.__file__ + assert tlz.functoolz.__file__ == toolz.functoolz.__file__ + + assert tlz.pipe is toolz.pipe + + assert 'tlz' in tlz.__doc__ + assert tlz.curried.__doc__ is not None diff --git a/etc/generate_curried.py b/etc/generate_curried.py deleted file mode 100755 index e4c164f..0000000 --- a/etc/generate_curried.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python -import cytoolz -import toolz -import toolz.curried - - -autogen_header = """\ -############################################################################### -# THIS FILE IS AUTOMATICALLY GENERATED # -# # -# Please make all edits in etc/generate_curried.py and re-run # -# cytoolz/tests/test_curried_toolzlike.py # -############################################################################### -""" - - -init_template = '''\ -""" -Alternate namespece for cytoolz such that all functions are curried - -Currying provides implicit partial evaluation of all functions - -Example: - - Get usually requires two arguments, an index and a collection - >>> from cytoolz.curried import get - >>> get(0, ('a', 'b')) - 'a' - - When we use it in higher order functions we often want to pass a partially - evaluated form - >>> data = [(1, 2), (11, 22), (111, 222)] - >>> list(map(lambda seq: get(0, seq), data)) - [1, 11, 111] - - The curried version allows simple expression of partial evaluation - >>> list(map(get(0), data)) - [1, 11, 111] - -See Also: - cytoolz.functoolz.curry -""" -from . import exceptions -from . import operator -import cytoolz - - -_curry_set = frozenset([ - {should_curry} -]) - - -def _curry_namespace(ns): - return dict( - (name, cytoolz.curry(f) if f in _curry_set else f) - for name, f in ns.items() if '__' not in name - ) - - -locals().update(cytoolz.merge( - _curry_namespace(vars(cytoolz)), - _curry_namespace(vars(exceptions)), -)) - -# Clean up the namespace. -del _curry_set -del _curry_namespace -del exceptions -del cytoolz -''' - - -def _curry_namespace(ns): - ct = vars(cytoolz) - return ( - 'cytoolz.' + name + ',' - for name, f in sorted(ns.items()) if isinstance(f, toolz.curry) and name in ct - ) - - -def gen_init(): - return init_template.format( - should_curry='\n '.join( - _curry_namespace(vars(toolz.curried)), - ), - ) - - -def main(argv): - if len(argv) != 1: - raise ValueError('no arguments expected') - - print(autogen_header + gen_init()) - - -if __name__ == '__main__': - from sys import argv - main(argv) From 0c3ed9d000d695b65c1b2e5a303e27d360dc4fdf Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sun, 4 Dec 2016 18:00:39 +0100 Subject: [PATCH 112/146] cytoolz would always import doctest, which is only used for testing --- cytoolz/tests/test_dicttoolz.py | 2 +- cytoolz/tests/test_docstrings.py | 2 +- cytoolz/tests/test_doctests.py | 2 +- cytoolz/tests/test_functoolz.py | 2 +- cytoolz/tests/test_inspect_args.py | 2 +- cytoolz/tests/test_itertoolz.py | 2 +- cytoolz/tests/test_none_safe.py | 2 +- cytoolz/tests/test_utils.py | 3 +- cytoolz/utils.pyx | 73 +---------------------------- cytoolz/utils_test.py | 74 ++++++++++++++++++++++++++++++ 10 files changed, 84 insertions(+), 80 deletions(-) create mode 100644 cytoolz/utils_test.py diff --git a/cytoolz/tests/test_dicttoolz.py b/cytoolz/tests/test_dicttoolz.py index 6327631..0775105 100644 --- a/cytoolz/tests/test_dicttoolz.py +++ b/cytoolz/tests/test_dicttoolz.py @@ -2,7 +2,7 @@ from cytoolz.dicttoolz import (merge, merge_with, valmap, keymap, update_in, assoc, dissoc, keyfilter, valfilter, itemmap, itemfilter, assoc_in) -from cytoolz.utils import raises +from cytoolz.utils_test import raises from cytoolz.compatibility import PY3 diff --git a/cytoolz/tests/test_docstrings.py b/cytoolz/tests/test_docstrings.py index 85654e6..f0c8303 100644 --- a/cytoolz/tests/test_docstrings.py +++ b/cytoolz/tests/test_docstrings.py @@ -2,7 +2,7 @@ import cytoolz from cytoolz import curry, identity, keyfilter, valfilter, merge_with -from cytoolz.utils import raises +from cytoolz.utils_test import raises from dev_skip_test import dev_skip_test diff --git a/cytoolz/tests/test_doctests.py b/cytoolz/tests/test_doctests.py index 8ca5e60..c2c0043 100644 --- a/cytoolz/tests/test_doctests.py +++ b/cytoolz/tests/test_doctests.py @@ -1,4 +1,4 @@ -from cytoolz.utils import module_doctest +from cytoolz.utils_test import module_doctest import cytoolz import cytoolz.dicttoolz diff --git a/cytoolz/tests/test_functoolz.py b/cytoolz/tests/test_functoolz.py index 5aa0723..f92ed3f 100644 --- a/cytoolz/tests/test_functoolz.py +++ b/cytoolz/tests/test_functoolz.py @@ -3,7 +3,7 @@ from cytoolz.functoolz import (thread_first, thread_last, memoize, curry, compose, pipe, complement, do, juxt, flip, excepts) from operator import add, mul, itemgetter -from cytoolz.utils import raises +from cytoolz.utils_test import raises from functools import partial diff --git a/cytoolz/tests/test_inspect_args.py b/cytoolz/tests/test_inspect_args.py index 061810f..dccc704 100644 --- a/cytoolz/tests/test_inspect_args.py +++ b/cytoolz/tests/test_inspect_args.py @@ -8,7 +8,7 @@ from cytoolz._signatures import builtins import cytoolz._signatures as _sigs from cytoolz.compatibility import PY3, PY33 -from cytoolz.utils import raises +from cytoolz.utils_test import raises def make_func(param_string, raise_if_called=True): diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index c7d99b8..886f5a2 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -1,6 +1,6 @@ import itertools from itertools import starmap -from cytoolz.utils import raises +from cytoolz.utils_test import raises from functools import partial from random import Random from pickle import dumps, loads diff --git a/cytoolz/tests/test_none_safe.py b/cytoolz/tests/test_none_safe.py index 8dbd7cc..f36e82f 100644 --- a/cytoolz/tests/test_none_safe.py +++ b/cytoolz/tests/test_none_safe.py @@ -20,7 +20,7 @@ # XXX: This file could be back-ported to `toolz` once unified testing exists. import cytoolz from cytoolz import * -from cytoolz.utils import raises +from cytoolz.utils_test import raises from operator import add diff --git a/cytoolz/tests/test_utils.py b/cytoolz/tests/test_utils.py index 0fabf4e..607b6a3 100644 --- a/cytoolz/tests/test_utils.py +++ b/cytoolz/tests/test_utils.py @@ -1,4 +1,5 @@ -from cytoolz.utils import raises, consume +from cytoolz.utils import consume +from cytoolz.utils_test import raises def test_raises(): diff --git a/cytoolz/utils.pyx b/cytoolz/utils.pyx index 8d39165..4b802aa 100644 --- a/cytoolz/utils.pyx +++ b/cytoolz/utils.pyx @@ -1,18 +1,8 @@ -import doctest -import inspect import os.path import cytoolz -__all__ = ['raises', 'no_default', 'include_dirs', 'consume', 'module_doctest'] - - -def raises(err, lamda): - try: - lamda() - return False - except err: - return True +__all__ = ['no_default', 'include_dirs', 'consume'] try: @@ -57,64 +47,3 @@ cpdef object consume(object seq): Efficiently consume an iterable """ for _ in seq: pass - - -# The utilities below were obtained from: -# https://github.com/cython/cython/wiki/FAQ -# #how-can-i-run-doctests-in-cython-code-pyx-files -# -# Cython-compatible wrapper for doctest.testmod(). -# -# Usage example, assuming a Cython module mymod.pyx is compiled. -# This is run from the command line, passing a command to Python: -# python -c "import cydoctest, mymod; cydoctest.testmod(mymod)" -# -# (This still won't let a Cython module run its own doctests -# when called with "python mymod.py", but it's pretty close. -# Further options can be passed to testmod() as desired, e.g. -# verbose=True.) - - -def _from_module(module, object): - """ - Return true if the given object is defined in the given module. - """ - if module is None: - return True - elif inspect.getmodule(object) is not None: - return module is inspect.getmodule(object) - elif inspect.isfunction(object): - return module.__dict__ is object.func_globals - elif inspect.isclass(object): - return module.__name__ == object.__module__ - elif hasattr(object, '__module__'): - return module.__name__ == object.__module__ - elif isinstance(object, property): - return True # [XX] no way not be sure. - else: - raise ValueError("object must be a class or function") - - -def _fix_module_doctest(module): - """ - Extract docstrings from cython functions, that would be skipped by doctest - otherwise. - """ - module.__test__ = {} - for name in dir(module): - value = getattr(module, name) - if (inspect.isbuiltin(value) and isinstance(value.__doc__, str) and - _from_module(module, value)): - module.__test__[name] = value.__doc__ - - -def module_doctest(m, *args, **kwargs): - """ - Fix a Cython module's doctests, then call doctest.testmod() - - All other arguments are passed directly to doctest.testmod(). - - Return True on success, False on failure. - """ - _fix_module_doctest(m) - return doctest.testmod(m, *args, **kwargs).failed == 0 diff --git a/cytoolz/utils_test.py b/cytoolz/utils_test.py new file mode 100644 index 0000000..1b517e4 --- /dev/null +++ b/cytoolz/utils_test.py @@ -0,0 +1,74 @@ +import doctest +import inspect + + +__all__ = ['raises', 'module_doctest'] + + +def raises(err, lamda): + try: + lamda() + return False + except err: + return True + + +# The utilities below were obtained from: +# https://github.com/cython/cython/wiki/FAQ +# #how-can-i-run-doctests-in-cython-code-pyx-files +# +# Cython-compatible wrapper for doctest.testmod(). +# +# Usage example, assuming a Cython module mymod.pyx is compiled. +# This is run from the command line, passing a command to Python: +# python -c "import cydoctest, mymod; cydoctest.testmod(mymod)" +# +# (This still won't let a Cython module run its own doctests +# when called with "python mymod.py", but it's pretty close. +# Further options can be passed to testmod() as desired, e.g. +# verbose=True.) + + +def _from_module(module, object): + """ + Return true if the given object is defined in the given module. + """ + if module is None: + return True + elif inspect.getmodule(object) is not None: + return module is inspect.getmodule(object) + elif inspect.isfunction(object): + return module.__dict__ is object.func_globals + elif inspect.isclass(object): + return module.__name__ == object.__module__ + elif hasattr(object, '__module__'): + return module.__name__ == object.__module__ + elif isinstance(object, property): + return True # [XX] no way not be sure. + else: + raise ValueError("object must be a class or function") + + +def _fix_module_doctest(module): + """ + Extract docstrings from cython functions, that would be skipped by doctest + otherwise. + """ + module.__test__ = {} + for name in dir(module): + value = getattr(module, name) + if (inspect.isbuiltin(value) and isinstance(value.__doc__, str) and + _from_module(module, value)): + module.__test__[name] = value.__doc__ + + +def module_doctest(m, *args, **kwargs): + """ + Fix a Cython module's doctests, then call doctest.testmod() + + All other arguments are passed directly to doctest.testmod(). + + Return True on success, False on failure. + """ + _fix_module_doctest(m) + return doctest.testmod(m, *args, **kwargs).failed == 0 From 1ea589a469217ac978b05adac013eebd908841b3 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sun, 4 Dec 2016 22:11:23 +0100 Subject: [PATCH 113/146] Put back `raises` in `cytoolz.utils` --- cytoolz/tests/test_dicttoolz.py | 2 +- cytoolz/tests/test_docstrings.py | 2 +- cytoolz/tests/test_functoolz.py | 2 +- cytoolz/tests/test_inspect_args.py | 2 +- cytoolz/tests/test_itertoolz.py | 2 +- cytoolz/tests/test_none_safe.py | 2 +- cytoolz/tests/test_utils.py | 3 +-- cytoolz/utils.pyx | 10 +++++++++- cytoolz/utils_test.py | 10 +--------- 9 files changed, 17 insertions(+), 18 deletions(-) diff --git a/cytoolz/tests/test_dicttoolz.py b/cytoolz/tests/test_dicttoolz.py index 0775105..6327631 100644 --- a/cytoolz/tests/test_dicttoolz.py +++ b/cytoolz/tests/test_dicttoolz.py @@ -2,7 +2,7 @@ from cytoolz.dicttoolz import (merge, merge_with, valmap, keymap, update_in, assoc, dissoc, keyfilter, valfilter, itemmap, itemfilter, assoc_in) -from cytoolz.utils_test import raises +from cytoolz.utils import raises from cytoolz.compatibility import PY3 diff --git a/cytoolz/tests/test_docstrings.py b/cytoolz/tests/test_docstrings.py index f0c8303..85654e6 100644 --- a/cytoolz/tests/test_docstrings.py +++ b/cytoolz/tests/test_docstrings.py @@ -2,7 +2,7 @@ import cytoolz from cytoolz import curry, identity, keyfilter, valfilter, merge_with -from cytoolz.utils_test import raises +from cytoolz.utils import raises from dev_skip_test import dev_skip_test diff --git a/cytoolz/tests/test_functoolz.py b/cytoolz/tests/test_functoolz.py index f92ed3f..5aa0723 100644 --- a/cytoolz/tests/test_functoolz.py +++ b/cytoolz/tests/test_functoolz.py @@ -3,7 +3,7 @@ from cytoolz.functoolz import (thread_first, thread_last, memoize, curry, compose, pipe, complement, do, juxt, flip, excepts) from operator import add, mul, itemgetter -from cytoolz.utils_test import raises +from cytoolz.utils import raises from functools import partial diff --git a/cytoolz/tests/test_inspect_args.py b/cytoolz/tests/test_inspect_args.py index dccc704..061810f 100644 --- a/cytoolz/tests/test_inspect_args.py +++ b/cytoolz/tests/test_inspect_args.py @@ -8,7 +8,7 @@ from cytoolz._signatures import builtins import cytoolz._signatures as _sigs from cytoolz.compatibility import PY3, PY33 -from cytoolz.utils_test import raises +from cytoolz.utils import raises def make_func(param_string, raise_if_called=True): diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index 886f5a2..c7d99b8 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -1,6 +1,6 @@ import itertools from itertools import starmap -from cytoolz.utils_test import raises +from cytoolz.utils import raises from functools import partial from random import Random from pickle import dumps, loads diff --git a/cytoolz/tests/test_none_safe.py b/cytoolz/tests/test_none_safe.py index f36e82f..8dbd7cc 100644 --- a/cytoolz/tests/test_none_safe.py +++ b/cytoolz/tests/test_none_safe.py @@ -20,7 +20,7 @@ # XXX: This file could be back-ported to `toolz` once unified testing exists. import cytoolz from cytoolz import * -from cytoolz.utils_test import raises +from cytoolz.utils import raises from operator import add diff --git a/cytoolz/tests/test_utils.py b/cytoolz/tests/test_utils.py index 607b6a3..2356b47 100644 --- a/cytoolz/tests/test_utils.py +++ b/cytoolz/tests/test_utils.py @@ -1,5 +1,4 @@ -from cytoolz.utils import consume -from cytoolz.utils_test import raises +from cytoolz.utils import consume, raises def test_raises(): diff --git a/cytoolz/utils.pyx b/cytoolz/utils.pyx index 4b802aa..010e50a 100644 --- a/cytoolz/utils.pyx +++ b/cytoolz/utils.pyx @@ -2,7 +2,7 @@ import os.path import cytoolz -__all__ = ['no_default', 'include_dirs', 'consume'] +__all__ = ['raises', 'no_default', 'include_dirs', 'consume'] try: @@ -12,6 +12,14 @@ except ImportError: no_default = '__no__default__' +def raises(err, lamda): + try: + lamda() + return False + except err: + return True + + def include_dirs(): """ Return a list of directories containing the *.pxd files for ``cytoolz`` diff --git a/cytoolz/utils_test.py b/cytoolz/utils_test.py index 1b517e4..418b352 100644 --- a/cytoolz/utils_test.py +++ b/cytoolz/utils_test.py @@ -2,15 +2,7 @@ import inspect -__all__ = ['raises', 'module_doctest'] - - -def raises(err, lamda): - try: - lamda() - return False - except err: - return True +__all__ = ['module_doctest'] # The utilities below were obtained from: From f15908daff3628da443c38d332758bfede8a418f Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 4 Dec 2016 20:22:02 -0600 Subject: [PATCH 114/146] Fix docstring. --- cytoolz/dicttoolz.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cytoolz/dicttoolz.pyx b/cytoolz/dicttoolz.pyx index 35c2f39..dabf920 100644 --- a/cytoolz/dicttoolz.pyx +++ b/cytoolz/dicttoolz.pyx @@ -383,7 +383,7 @@ cpdef object assoc_in(object d, object keys, object value, object factory=dict): >>> assoc_in(purchase, ['order', 'costs'], [0.25, 1.00]) # doctest: +SKIP {'credit card': '5555-1234-1234-1234', 'name': 'Alice', - 'purchase': {'costs': [0.25, 1.00], 'items': ['Apple', 'Orange']}} + 'order': {'costs': [0.25, 1.00], 'items': ['Apple', 'Orange']}} """ cdef object prevkey, key cdef object rv, inner, dtemp From 8a253433002eea01cbb0156397d573bc5991048e Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 4 Dec 2016 21:12:07 -0600 Subject: [PATCH 115/146] Skip checking signature matches toolz for memoize and flip. There are differences in Python <3.3, and it's probably not worth the hassle. --- cytoolz/tests/test_embedded_sigs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cytoolz/tests/test_embedded_sigs.py b/cytoolz/tests/test_embedded_sigs.py index b360c91..7eed654 100644 --- a/cytoolz/tests/test_embedded_sigs.py +++ b/cytoolz/tests/test_embedded_sigs.py @@ -32,7 +32,7 @@ def test_class_sigs(): d = merge_with(identity, toolz_dict, cytoolz_dict) for key, (toolz_func, cytoolz_func) in d.items(): - if key in ['excepts', 'juxt']: + if key in ['excepts', 'juxt', 'memoize', 'flip']: continue try: # function From f5e7865f43fe007805ca583f4e95d3faea15c051 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 4 Dec 2016 21:18:07 -0600 Subject: [PATCH 116/146] Bump to next dev version, 0.8.2dev --- conda.recipe/meta.yaml | 2 +- cytoolz/_version.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index c7eb3fc..c3896ce 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: cytoolz - version: "0.8.0" + version: "0.8.1" build: number: {{environ.get('BINSTAR_BUILD', 1)}} diff --git a/cytoolz/_version.py b/cytoolz/_version.py index 7805a97..0b96ad5 100644 --- a/cytoolz/_version.py +++ b/cytoolz/_version.py @@ -1,2 +1,2 @@ -__version__ = '0.8.1dev' -__toolz_version__ = '0.8.0' +__version__ = '0.8.2dev' +__toolz_version__ = '0.8.1' From c06d239d92a11775034c028c1414a26fea26433a Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 11 Dec 2016 15:16:49 -0600 Subject: [PATCH 117/146] Fix serializing global curried objects. pytoolz/toolz#355 Also, fix signature registry for flip and memoize. I had forgotten that we can list multiple signatures. --- Makefile | 1 + cytoolz/_signatures.py | 12 ++- cytoolz/functoolz.pxd | 2 + cytoolz/functoolz.pyx | 46 +++++++--- cytoolz/tests/test_functoolz.py | 12 ++- cytoolz/tests/test_serialization.py | 138 ++++++++++++++++++++++++++++ 6 files changed, 191 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index e01bc95..d6cc7e0 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ +SHELL= /bin/bash PYTHON ?= python inplace: diff --git a/cytoolz/_signatures.py b/cytoolz/_signatures.py index cea1de6..cc5a3c4 100644 --- a/cytoolz/_signatures.py +++ b/cytoolz/_signatures.py @@ -46,16 +46,20 @@ lambda func, x: None], excepts=[ lambda exc, func, handler=None: None], - flip=[ # XXX: these are optional, but not keywords! - lambda func=None, a=None, b=None: None], + flip=[ + lambda: None, + lambda func: None, + lambda func, a: None, + lambda func, a, b: None], _flip=[ lambda func, a, b: None], identity=[ lambda x: None], juxt=[ lambda *funcs: None], - memoize=[ # XXX: func is optional, but not a keyword! - lambda func=None, cache=None, key=None: None], + memoize=[ + lambda cache=None, key=None: None, + lambda func, cache=None, key=None: None], _memoize=[ lambda func, cache=None, key=None: None], pipe=[ diff --git a/cytoolz/functoolz.pxd b/cytoolz/functoolz.pxd index 0eee035..766aca1 100644 --- a/cytoolz/functoolz.pxd +++ b/cytoolz/functoolz.pxd @@ -15,6 +15,8 @@ cdef class curry: cdef readonly dict keywords cdef public object __doc__ cdef public object __name__ + cdef public object __module__ + cdef public object __qualname__ cdef class memoize: cdef object func diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 54a2ba4..975a6b9 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -194,6 +194,8 @@ cdef class curry: self.keywords = kwargs if kwargs else _empty_kwargs() self.__doc__ = getattr(func, '__doc__', None) self.__name__ = getattr(func, '__name__', '') + self.__module__ = getattr(func, '__module__', None) + self.__qualname__ = getattr(func, '__qualname__', None) self._sigspec = None self._has_unknown_args = None @@ -331,29 +333,43 @@ cdef class curry: def __reduce__(self): func = self.func modname = getattr(func, '__module__', None) - funcname = getattr(func, '__name__', None) - if modname and funcname: - module = import_module(modname) - obj = getattr(module, funcname, None) - if obj is self: - return funcname - elif isinstance(obj, curry) and obj.func is func: - func = '%s.%s' % (modname, funcname) - - state = (type(self), func, self.args, self.keywords) + qualname = getattr(func, '__qualname__', None) + if qualname is None: + qualname = getattr(func, '__name__', None) + is_decorated = None + if modname and qualname: + attrs = [] + obj = import_module(modname) + for attr in qualname.split('.'): + if isinstance(obj, curry): + attrs.append('func') + obj = obj.func + obj = getattr(obj, attr, None) + if obj is None: + break + attrs.append(attr) + if isinstance(obj, curry) and obj.func is func: + is_decorated = obj is self + qualname = '.'.join(attrs) + func = '%s:%s' % (modname, qualname) + + state = (type(self), func, self.args, self.keywords, is_decorated) return (_restore_curry, state) -cpdef object _restore_curry(cls, func, args, kwargs): +cpdef object _restore_curry(cls, func, args, kwargs, is_decorated): if isinstance(func, str): - modname, funcname = func.rsplit('.', 1) - module = import_module(modname) - func = getattr(module, funcname).func + modname, qualname = func.rsplit(':', 1) + obj = import_module(modname) + for attr in qualname.split('.'): + obj = getattr(obj, attr) + if is_decorated: + return obj + func = obj.func obj = cls(func, *args, **(kwargs or {})) return obj - cdef class memoize: """ memoize(func, cache=None, key=None) diff --git a/cytoolz/tests/test_functoolz.py b/cytoolz/tests/test_functoolz.py index 5aa0723..b6ddcf1 100644 --- a/cytoolz/tests/test_functoolz.py +++ b/cytoolz/tests/test_functoolz.py @@ -285,16 +285,26 @@ def foo(a, b, c=1): def test_curry_attributes_writable(): def foo(a, b, c=1): return a + b + c - + foo.__qualname__ = 'this.is.foo' f = curry(foo, 1, c=2) + assert f.__qualname__ == 'this.is.foo' f.__name__ = 'newname' f.__doc__ = 'newdoc' + f.__module__ = 'newmodule' + f.__qualname__ = 'newqualname' assert f.__name__ == 'newname' assert f.__doc__ == 'newdoc' + assert f.__module__ == 'newmodule' + assert f.__qualname__ == 'newqualname' if hasattr(f, 'func_name'): assert f.__name__ == f.func_name +def test_curry_module(): + from cytoolz.curried.exceptions import merge + assert merge.__module__ == 'cytoolz.curried.exceptions' + + def test_curry_comparable(): def foo(a, b, c=1): return a + b + c diff --git a/cytoolz/tests/test_serialization.py b/cytoolz/tests/test_serialization.py index f5522b4..be216f8 100644 --- a/cytoolz/tests/test_serialization.py +++ b/cytoolz/tests/test_serialization.py @@ -1,6 +1,9 @@ from cytoolz import * import cytoolz +import cytoolz.curried.exceptions import pickle +from cytoolz.compatibility import PY3, PY33, PY34 +from cytoolz.utils import raises def test_compose(): @@ -55,3 +58,138 @@ def test_flip(): g1 = flip(f)(1) g2 = pickle.loads(pickle.dumps(g1)) assert g1(2) == g2(2) == f(2, 1) + + +def test_curried_exceptions(): + # This tests a global curried object that isn't defined in cytoolz.functoolz + merge = pickle.loads(pickle.dumps(cytoolz.curried.exceptions.merge)) + assert merge is cytoolz.curried.exceptions.merge + + +@cytoolz.curry +class GlobalCurried(object): + def __init__(self, x, y): + self.x = x + self.y = y + + @cytoolz.curry + def f1(self, a, b): + return self.x + self.y + a + b + + def g1(self): + pass + + def __reduce__(self): + """Allow us to serialize instances of GlobalCurried""" + return (GlobalCurried, (self.x, self.y)) + + @cytoolz.curry + class NestedCurried(object): + def __init__(self, x, y): + self.x = x + self.y = y + + @cytoolz.curry + def f2(self, a, b): + return self.x + self.y + a + b + + def g2(self): + pass + + def __reduce__(self): + """Allow us to serialize instances of NestedCurried""" + return (GlobalCurried.NestedCurried, (self.x, self.y)) + + class Nested(object): + def __init__(self, x, y): + self.x = x + self.y = y + + @cytoolz.curry + def f3(self, a, b): + return self.x + self.y + a + b + + def g3(self): + pass + + +def test_curried_qualname(): + if not PY3: + return + + def preserves_identity(obj): + return pickle.loads(pickle.dumps(obj)) is obj + + assert preserves_identity(GlobalCurried) + assert preserves_identity(GlobalCurried.func.f1) + assert preserves_identity(GlobalCurried.func.NestedCurried) + assert preserves_identity(GlobalCurried.func.NestedCurried.func.f2) + assert preserves_identity(GlobalCurried.func.Nested.f3) + + global_curried1 = GlobalCurried(1) + global_curried2 = pickle.loads(pickle.dumps(global_curried1)) + assert global_curried1 is not global_curried2 + assert global_curried1(2).f1(3, 4) == global_curried2(2).f1(3, 4) == 10 + + global_curried3 = global_curried1(2) + global_curried4 = pickle.loads(pickle.dumps(global_curried3)) + assert global_curried3 is not global_curried4 + assert global_curried3.f1(3, 4) == global_curried4.f1(3, 4) == 10 + + func1 = global_curried1(2).f1(3) + func2 = pickle.loads(pickle.dumps(func1)) + assert func1 is not func2 + assert func1(4) == func2(4) == 10 + + nested_curried1 = GlobalCurried.func.NestedCurried(1) + nested_curried2 = pickle.loads(pickle.dumps(nested_curried1)) + assert nested_curried1 is not nested_curried2 + assert nested_curried1(2).f2(3, 4) == nested_curried2(2).f2(3, 4) == 10 + + # If we add `curry.__getattr__` forwarding, the following tests will pass + + # if not PY33 and not PY34: + # assert preserves_identity(GlobalCurried.func.g1) + # assert preserves_identity(GlobalCurried.func.NestedCurried.func.g2) + # assert preserves_identity(GlobalCurried.func.Nested) + # assert preserves_identity(GlobalCurried.func.Nested.g3) + # + # # Rely on curry.__getattr__ + # assert preserves_identity(GlobalCurried.f1) + # assert preserves_identity(GlobalCurried.NestedCurried) + # assert preserves_identity(GlobalCurried.NestedCurried.f2) + # assert preserves_identity(GlobalCurried.Nested.f3) + # if not PY33 and not PY34: + # assert preserves_identity(GlobalCurried.g1) + # assert preserves_identity(GlobalCurried.NestedCurried.g2) + # assert preserves_identity(GlobalCurried.Nested) + # assert preserves_identity(GlobalCurried.Nested.g3) + # + # nested_curried3 = nested_curried1(2) + # nested_curried4 = pickle.loads(pickle.dumps(nested_curried3)) + # assert nested_curried3 is not nested_curried4 + # assert nested_curried3.f2(3, 4) == nested_curried4.f2(3, 4) == 10 + # + # func1 = nested_curried1(2).f2(3) + # func2 = pickle.loads(pickle.dumps(func1)) + # assert func1 is not func2 + # assert func1(4) == func2(4) == 10 + # + # if not PY33 and not PY34: + # nested3 = GlobalCurried.func.Nested(1, 2) + # nested4 = pickle.loads(pickle.dumps(nested3)) + # assert nested3 is not nested4 + # assert nested3.f3(3, 4) == nested4.f3(3, 4) == 10 + # + # func1 = nested3.f3(3) + # func2 = pickle.loads(pickle.dumps(func1)) + # assert func1 is not func2 + # assert func1(4) == func2(4) == 10 + + +def test_curried_bad_qualname(): + @cytoolz.curry + class Bad(object): + __qualname__ = 'cytoolz.functoolz.not.a.valid.path' + + assert raises(pickle.PicklingError, lambda: pickle.dumps(Bad)) From 9634ec83ef90edc7e8da551cd0847dc07f88d893 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 11 Dec 2016 16:03:10 -0600 Subject: [PATCH 118/146] Adding `__module__` to `curry` made `curry.__signature__` break in Python 3.3 ...because of course Python 3.3 is different. --- cytoolz/functoolz.pyx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 975a6b9..2f9449b 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -4,7 +4,7 @@ from functools import partial from operator import attrgetter from textwrap import dedent from cytoolz.utils import no_default -from cytoolz.compatibility import PY3, PY34, filter as ifilter, map as imap, reduce, import_module +from cytoolz.compatibility import PY3, PY33, PY34, filter as ifilter, map as imap, reduce, import_module import cytoolz._signatures as _sigs from toolz.functoolz import (InstanceProperty, instanceproperty, is_arity, @@ -294,7 +294,13 @@ cdef class curry: property __signature__: def __get__(self): - sig = inspect.signature(self.func) + try: + sig = inspect.signature(self.func) + except TypeError: + if PY33 and (getattr(self.func, '__module__') or '').startswith('cytoolz.'): + raise ValueError('callable %r is not supported by signature' % self.func) + raise + args = self.args or () keywords = self.keywords or {} if is_partial_args(self.func, args, keywords, sig) is False: From 7470fdd80f0b952422118f8bb8bfb0cb55d66d68 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 11 Dec 2016 16:33:07 -0600 Subject: [PATCH 119/146] Update badges to use svg and match what toolz does. --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 6af7dc3..86c2274 100644 --- a/README.rst +++ b/README.rst @@ -72,9 +72,9 @@ Community See our `mailing list `__. We're friendly. -.. |Build Status| image:: https://travis-ci.org/pytoolz/cytoolz.png +.. |Build Status| image:: https://travis-ci.org/pytoolz/cytoolz.svg?branch=master :target: https://travis-ci.org/pytoolz/cytoolz -.. |Version Status| image:: https://pypip.in/v/cytoolz/badge.png - :target: https://pypi.python.org/pypi/cytoolz/ -.. |Downloads| image:: https://pypip.in/d/cytoolz/badge.png +.. |Version Status| image:: https://badge.fury.io/py/cytoolz.svg + :target: http://badge.fury.io/py/cytoolz +.. |Downloads| image:: https://img.shields.io/pypi/dm/cytoolz.svg :target: https://pypi.python.org/pypi/cytoolz/ From b8f1ef602d0851092fe3010f9f952aeccbeddd31 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 11 Dec 2016 17:23:09 -0600 Subject: [PATCH 120/146] Bump to next dev version. 0.8.2 just released. --- conda.recipe/meta.yaml | 2 +- cytoolz/_version.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index c3896ce..d6cb80f 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: cytoolz - version: "0.8.1" + version: "0.8.2" build: number: {{environ.get('BINSTAR_BUILD', 1)}} diff --git a/cytoolz/_version.py b/cytoolz/_version.py index 0b96ad5..9357354 100644 --- a/cytoolz/_version.py +++ b/cytoolz/_version.py @@ -1,2 +1,2 @@ -__version__ = '0.8.2dev' -__toolz_version__ = '0.8.1' +__version__ = '0.8.3dev' +__toolz_version__ = '0.8.2' From 6fc6fae73ef65eadd96d90f7a8ed3f8e5e82ca39 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 16 Jan 2017 00:13:45 -0600 Subject: [PATCH 121/146] Add python 3.6 to the test matrix --- .travis.yml | 1 + setup.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 37137fc..f043c05 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - "3.3" - "3.4" - "3.5" + - "3.6" before_install: - pip install git+https://github.com/pytoolz/toolz.git diff --git a/setup.py b/setup.py index 014844a..1a5f588 100644 --- a/setup.py +++ b/setup.py @@ -90,7 +90,7 @@ keywords=('functional utility itertools functools iterator generator ' 'curry memoize lazy streaming bigdata cython toolz cytoolz'), classifiers = [ - 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Intended Audience :: Education', 'Intended Audience :: Science/Research', @@ -105,6 +105,7 @@ 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Scientific/Engineering', 'Topic :: Scientific/Engineering :: Information Analysis', 'Topic :: Software Development', From ce207df684678c7fe4e5df011f65997923f91abd Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 4 Feb 2017 13:45:25 -0600 Subject: [PATCH 122/146] Make sure bare test commands succeed. #97 --- .travis.yml | 5 ++++- cytoolz/tests/test_serialization.py | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f043c05..6d13122 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ python: before_install: - pip install git+https://github.com/pytoolz/toolz.git - - pip install cython + - pip install cython pytest install: - python setup.py build_ext --inplace --with-cython @@ -17,6 +17,9 @@ install: # commands to run tests script: - nosetests --with-doctest cytoolz + # For convenience, make sure simple test commands work + - py.test + - nosetests notifications: email: false diff --git a/cytoolz/tests/test_serialization.py b/cytoolz/tests/test_serialization.py index be216f8..7f95cbe 100644 --- a/cytoolz/tests/test_serialization.py +++ b/cytoolz/tests/test_serialization.py @@ -1,6 +1,6 @@ from cytoolz import * import cytoolz -import cytoolz.curried.exceptions +import cytoolz.curried import pickle from cytoolz.compatibility import PY3, PY33, PY34 from cytoolz.utils import raises @@ -62,8 +62,8 @@ def test_flip(): def test_curried_exceptions(): # This tests a global curried object that isn't defined in cytoolz.functoolz - merge = pickle.loads(pickle.dumps(cytoolz.curried.exceptions.merge)) - assert merge is cytoolz.curried.exceptions.merge + merge = pickle.loads(pickle.dumps(cytoolz.curried.merge)) + assert merge is cytoolz.curried.merge @cytoolz.curry From d06f814b8d4dd99fc2660f8a95e0af1d54a2aa8f Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 4 Feb 2017 13:50:37 -0600 Subject: [PATCH 123/146] Oups indentation. --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6d13122..f1a2470 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,9 +17,9 @@ install: # commands to run tests script: - nosetests --with-doctest cytoolz - # For convenience, make sure simple test commands work - - py.test - - nosetests + # For convenience, make sure simple test commands work + - py.test + - nosetests notifications: email: false From e6f14cf27c1b2d9cb7d552cfdc0bf73cd020ab6a Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 26 Sep 2017 15:45:43 -0400 Subject: [PATCH 124/146] Fix #104. Be more careful when getting references without the Cython refnanny! Notably, `get` would fail when the gotten object had exactly 1 reference count. This is actually uncommon in Python, but it can happen in Numpy (or when Python objects are managed in C). --- cytoolz/itertoolz.pyx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 262619c..9c4f7b1 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -687,8 +687,9 @@ cpdef object get(object ind, object seq, object default='__no__default__'): if PyErr_GivenExceptionMatches(val, _get_exceptions): return default raise val + val = obj Py_XDECREF(obj) - return obj + return val cpdef object concat(object seqs): @@ -1107,8 +1108,9 @@ cdef class _pluck_index_default: if not PyErr_GivenExceptionMatches(val, _get_exceptions): raise val return self.default + val = obj Py_XDECREF(obj) - return obj + return val cdef class _pluck_list: From bff5bc874b007378c40913d3d3675639aec95344 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 26 Sep 2017 16:00:58 -0400 Subject: [PATCH 125/146] Don't require introspection on `__reduce_cython__` (and similar) functions. TODO: copy this to `toolz` so `make copytests` will work. --- cytoolz/tests/test_inspect_args.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cytoolz/tests/test_inspect_args.py b/cytoolz/tests/test_inspect_args.py index 061810f..a5371eb 100644 --- a/cytoolz/tests/test_inspect_args.py +++ b/cytoolz/tests/test_inspect_args.py @@ -409,6 +409,8 @@ def add_blacklist(mod, attr): def is_missing(modname, name, func): if name.startswith('_') and not name.startswith('__'): return False + if name.startswith('__pyx_unpickle_') or name.endswith('_cython__'): + return False try: if issubclass(func, BaseException): return False From f7229426c6d0e50916bbae4a63b4235fb2f6637c Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Thu, 30 Nov 2017 14:19:32 -0500 Subject: [PATCH 126/146] BUG: add __module__ to compose --- cytoolz/functoolz.pyx | 3 +++ cytoolz/tests/test_functoolz.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 2f9449b..a1ba4e0 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -479,6 +479,9 @@ cdef class Compose: See Also: compose """ + # fix for #103, note: we cannot use __name__ at module-scope in cython + __module__ = 'cytooz.functoolz' + def __cinit__(self, *funcs): self.first = funcs[-1] self.funcs = tuple(reversed(funcs[:-1])) diff --git a/cytoolz/tests/test_functoolz.py b/cytoolz/tests/test_functoolz.py index b6ddcf1..1dd997a 100644 --- a/cytoolz/tests/test_functoolz.py +++ b/cytoolz/tests/test_functoolz.py @@ -530,6 +530,11 @@ def g(a): assert composed.__doc__ == 'A composition of functions' +def test_compose_module(): + # regression test for #103 + assert compose().__module__ == 'cytoolz.functoolz' + + def test_pipe(): assert pipe(1, inc) == 2 assert pipe(1, inc, inc) == 3 From d4535b52ec4e9e298d2624e71e92d8905320bce1 Mon Sep 17 00:00:00 2001 From: anonymous Date: Sat, 16 Dec 2017 00:03:25 +0000 Subject: [PATCH 127/146] preliminary debian package work --- debian/changelog | 5 +++++ debian/compat | 1 + debian/control | 20 +++++++++++++++++ debian/copyright | 37 ++++++++++++++++++++++++++++++++ debian/files | 3 +++ debian/python3-cytoolz.substvars | 6 ++++++ debian/rules | 10 +++++++++ 7 files changed, 82 insertions(+) create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/files create mode 100644 debian/python3-cytoolz.substvars create mode 100755 debian/rules diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..6a8dc89 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +python3-cytoolz (0.8.2-1) UNRELEASED; urgency=high + + * new package + + -- Jeff Cliff Sat, 25 Nov 2017 14:16:28 +0500 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +9 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..4bcdf37 --- /dev/null +++ b/debian/control @@ -0,0 +1,20 @@ +# ------------------------------------------------------------------------- # +# DEBIAN PACKAGE CONTROL FILE # +# # +# This file is a Debian control file. For more information on the config in # +# this file, please run `man deb-control`. # +# ------------------------------------------------------------------------- # + +Source: python3-cytoolz +Section: contrib/python +Priority: extra +Maintainer: Jeff Cliff +Build-Depends: debhelper(>= 9), cython3, python3, dh-python, python3-nose +Standards-Version: 3.9.5 + +Package: python3-cytoolz +Architecture: any +Pre-Depends: dpkg (>= 1.16.1), cython3, python3 +Depends: make, python3-nose, ${shlibs:Depends} +Description: Cython implementation of Toolz + High performance functional utilities diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..ea58cd3 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,37 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: cytoolz +Source: https://github.com/pytoolz/cytoolz + +Files: * +Copyright: 2014-2017 Erik Welch +License: 3-clause BSD +Copyright (c) 2014 Erik Welch + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + a. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + c. Neither the name of cytoolz nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. + + diff --git a/debian/files b/debian/files new file mode 100644 index 0000000..0205bd6 --- /dev/null +++ b/debian/files @@ -0,0 +1,3 @@ +python3-cytoolz-dbgsym_0.8.2-1_amd64.ddeb contrib/debug optional +python3-cytoolz_0.8.2-1_amd64.buildinfo contrib/python extra +python3-cytoolz_0.8.2-1_amd64.deb contrib/python extra diff --git a/debian/python3-cytoolz.substvars b/debian/python3-cytoolz.substvars new file mode 100644 index 0000000..29075f4 --- /dev/null +++ b/debian/python3-cytoolz.substvars @@ -0,0 +1,6 @@ +python3:Depends=python3 (<< 3.7), python3 (>= 3.6~), python3-toolz, python3:any (>= 3.3.2-2~) +python3:Provides=python3.6-cytoolz +python3:Versions=3.6 +shlibs:Depends=libc6 (>= 2.4) +misc:Depends= +misc:Pre-Depends= diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..70cb7da --- /dev/null +++ b/debian/rules @@ -0,0 +1,10 @@ +#! /usr/bin/make -f + +export DH_VERBOSE = 1 +export PYBUILD_NAME = foo + +%: + dh $@ --with python3 --buildsystem=pybuild + + + From 1d78e85d233438714acebc9d17bb309b38d2fc46 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 16 Dec 2017 12:14:00 -0600 Subject: [PATCH 128/146] Fix #101. Modify how we handle curried `memoize` to make `cimport cytoolz` work again. Also added a test to make sure we can cimport cytoolz. --- .travis.yml | 4 ++++ Makefile | 4 ++++ cytoolz/functoolz.pxd | 6 +++++- cytoolz/functoolz.pyx | 14 +++++++------- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index f1a2470..7b607e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,10 @@ script: # For convenience, make sure simple test commands work - py.test - nosetests + # Make sure we can cimport cytoolz + - echo 'cimport cytoolz ; from cytoolz.functoolz cimport memoize' > try_cimport_cytoolz.pyx + - cythonize -i try_cimport_cytoolz.pyx + - python -c 'import try_cimport_cytoolz' notifications: email: false diff --git a/Makefile b/Makefile index d6cc7e0..b3aa597 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,10 @@ inplace: test: inplace nosetests -s --with-doctest cytoolz/ + echo 'cimport cytoolz ; from cytoolz.functoolz cimport memoize' > try_cimport_cytoolz.pyx + cythonize -i try_cimport_cytoolz.pyx + python -c 'import try_cimport_cytoolz' + rm try_cimport_cytoolz.pyx clean: rm -f cytoolz/*.c cytoolz/*.so cytoolz/*/*.c cytoolz/*/*.so diff --git a/cytoolz/functoolz.pxd b/cytoolz/functoolz.pxd index 766aca1..6ee89f6 100644 --- a/cytoolz/functoolz.pxd +++ b/cytoolz/functoolz.pxd @@ -18,7 +18,11 @@ cdef class curry: cdef public object __module__ cdef public object __qualname__ -cdef class memoize: + +cpdef object memoize(object func, object cache=*, object key=*) + + +cdef class _memoize: cdef object func cdef object cache cdef object key diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index a1ba4e0..8e4baa5 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -376,9 +376,8 @@ cpdef object _restore_curry(cls, func, args, kwargs, is_decorated): return obj -cdef class memoize: - """ memoize(func, cache=None, key=None) - +cpdef object memoize(object func, object cache=None, object key=None): + """ Cache a function's result for speedy future evaluation Considerations: @@ -414,6 +413,10 @@ cdef class memoize: ... print('Calculating %s + %s' % (x, y)) ... return x + y """ + return _memoize(func, cache, key) + + +cdef class _memoize: property __doc__: def __get__(self): @@ -427,7 +430,7 @@ cdef class memoize: def __get__(self): return self.func - def __cinit__(self, func, cache=None, key=None): + def __cinit__(self, func, cache, key): self.func = func if cache is None: self.cache = PyDict_New() @@ -468,9 +471,6 @@ cdef class memoize: return curry(self, instance) -_memoize = memoize # uncurried - - cdef class Compose: """ Compose(self, *funcs) From 79ddc4843c16828f7d7dc626c8e176ed9be4ff27 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 16 Dec 2017 12:20:44 -0600 Subject: [PATCH 129/146] test python 3.7 in travi-ci --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index f1a2470..661972e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ python: - "3.4" - "3.5" - "3.6" + - "3.7-dev" before_install: - pip install git+https://github.com/pytoolz/toolz.git From 927b688407062db9e6c0cc67defb4750ac2fa3e9 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 16 Dec 2017 12:44:18 -0600 Subject: [PATCH 130/146] Allow tests to fail on Python 3.7-dev (for now) --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 661972e..ba6412a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,10 @@ python: - "3.6" - "3.7-dev" +matrix: + allow_failures: + - python: "3.7-dev" + before_install: - pip install git+https://github.com/pytoolz/toolz.git - pip install cython pytest From 9f8e39dcdbd676f740b654d4af87532cfd6fe5c0 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 16 Dec 2017 23:10:40 -0600 Subject: [PATCH 131/146] Update README. PyPI no longer provides download count badge. Also, Python 3.3 is the minimum version of Python 3 we test and support. --- Makefile | 2 +- README.rst | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index b3aa597..d50c388 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ test: inplace echo 'cimport cytoolz ; from cytoolz.functoolz cimport memoize' > try_cimport_cytoolz.pyx cythonize -i try_cimport_cytoolz.pyx python -c 'import try_cimport_cytoolz' - rm try_cimport_cytoolz.pyx + rm try_cimport_cytoolz.* clean: rm -f cytoolz/*.c cytoolz/*.so cytoolz/*/*.c cytoolz/*/*.so diff --git a/README.rst b/README.rst index 86c2274..703cb8a 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ CyToolz ======= -|Build Status| |Version Status| |Downloads| +|Build Status| |Version Status| Cython implementation of the |literal toolz|_ `package, `__ which @@ -48,7 +48,7 @@ Install Dependencies ------------ -``cytoolz`` supports Python 2.6+ and Python 3.2+ with a common codebase. +``cytoolz`` supports Python 2.6+ and Python 3.3+ with a common codebase. It is developed in Cython, but requires no dependecies other than CPython and a C compiler. Like ``toolz``, it is a light weight dependency. @@ -76,5 +76,3 @@ We're friendly. :target: https://travis-ci.org/pytoolz/cytoolz .. |Version Status| image:: https://badge.fury.io/py/cytoolz.svg :target: http://badge.fury.io/py/cytoolz -.. |Downloads| image:: https://img.shields.io/pypi/dm/cytoolz.svg - :target: https://pypi.python.org/pypi/cytoolz/ From 301ea9eb3ba0c4d7d69198cb8eea15a1d2116894 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 17 Dec 2017 11:17:03 -0600 Subject: [PATCH 132/146] Just released 0.9.0. Bump to next dev version, 0.9.1dev. --- conda.recipe/meta.yaml | 2 +- cytoolz/_version.py | 4 ++-- cytoolz/tests/test_inspect_args.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index d6cb80f..861d82b 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: cytoolz - version: "0.8.2" + version: "0.9.0" build: number: {{environ.get('BINSTAR_BUILD', 1)}} diff --git a/cytoolz/_version.py b/cytoolz/_version.py index 9357354..c0b02e7 100644 --- a/cytoolz/_version.py +++ b/cytoolz/_version.py @@ -1,2 +1,2 @@ -__version__ = '0.8.3dev' -__toolz_version__ = '0.8.2' +__version__ = '0.9.1dev' +__toolz_version__ = '0.9.0' diff --git a/cytoolz/tests/test_inspect_args.py b/cytoolz/tests/test_inspect_args.py index a5371eb..17b64ad 100644 --- a/cytoolz/tests/test_inspect_args.py +++ b/cytoolz/tests/test_inspect_args.py @@ -402,6 +402,7 @@ def add_blacklist(mod, attr): blacklist.add(getattr(mod, attr)) add_blacklist(builtins, 'basestring') + add_blacklist(builtins, 'breakpoint') add_blacklist(builtins, 'NoneType') add_blacklist(builtins, '__metaclass__') add_blacklist(builtins, 'sequenceiterator') @@ -497,4 +498,3 @@ def __wrapped__(self): assert num_required_args(Wrapped) == (False if PY33 else None) _sigs.signatures[Wrapped] = (_sigs.expand_sig((0, lambda func: None)),) assert num_required_args(Wrapped) == 1 - From 28315328fa074b54b7c9a7ceac55f7e44346912f Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 9 Jul 2019 16:16:47 -0500 Subject: [PATCH 133/146] Update to match latest toolz --- .travis.yml | 9 +- cytoolz/compatibility.py | 1 - cytoolz/curried/__init__.py | 4 + cytoolz/dicttoolz.pxd | 2 +- cytoolz/dicttoolz.pyx | 24 ++-- cytoolz/functoolz.pxd | 3 + cytoolz/functoolz.pyx | 100 +++++++++++++-- cytoolz/itertoolz.pxd | 3 + cytoolz/itertoolz.pyx | 30 ++++- cytoolz/tests/test_curried.py | 10 +- cytoolz/tests/test_dicttoolz.py | 12 +- cytoolz/tests/test_functoolz.py | 189 ++++++++++++++++++++++++---- cytoolz/tests/test_inspect_args.py | 6 +- cytoolz/tests/test_itertoolz.py | 61 ++++++--- cytoolz/tests/test_none_safe.py | 12 ++ cytoolz/tests/test_serialization.py | 12 +- 16 files changed, 391 insertions(+), 87 deletions(-) diff --git a/.travis.yml b/.travis.yml index e5ce07e..40a1b37 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,17 @@ language: python python: - - "2.6" - "2.7" - - "3.3" - "3.4" - "3.5" - "3.6" - "3.7-dev" +# Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs matrix: - allow_failures: - - python: "3.7-dev" + include: + - python: 3.7 + dist: xenial + sudo: true before_install: - pip install git+https://github.com/pytoolz/toolz.git diff --git a/cytoolz/compatibility.py b/cytoolz/compatibility.py index 595d43e..65af3f2 100644 --- a/cytoolz/compatibility.py +++ b/cytoolz/compatibility.py @@ -1,7 +1,6 @@ import operator import sys PY3 = sys.version_info[0] > 2 -PY33 = sys.version_info[0] == 3 and sys.version_info[1] == 3 PY34 = sys.version_info[0] == 3 and sys.version_info[1] == 4 __all__ = ['PY3', 'map', 'filter', 'range', 'zip', 'reduce', 'zip_longest', diff --git a/cytoolz/curried/__init__.py b/cytoolz/curried/__init__.py index a27783d..281f62f 100644 --- a/cytoolz/curried/__init__.py +++ b/cytoolz/curried/__init__.py @@ -26,9 +26,11 @@ import cytoolz from . import operator from cytoolz import ( + apply, comp, complement, compose, + compose_left, concat, concatv, count, @@ -59,6 +61,7 @@ assoc_in = cytoolz.curry(cytoolz.assoc_in) cons = cytoolz.curry(cytoolz.cons) countby = cytoolz.curry(cytoolz.countby) +dissoc = cytoolz.curry(cytoolz.dissoc) do = cytoolz.curry(cytoolz.do) drop = cytoolz.curry(cytoolz.drop) excepts = cytoolz.curry(cytoolz.excepts) @@ -80,6 +83,7 @@ partition = cytoolz.curry(cytoolz.partition) partition_all = cytoolz.curry(cytoolz.partition_all) partitionby = cytoolz.curry(cytoolz.partitionby) +peekn = cytoolz.curry(cytoolz.peekn) pluck = cytoolz.curry(cytoolz.pluck) random_sample = cytoolz.curry(cytoolz.random_sample) reduce = cytoolz.curry(cytoolz.reduce) diff --git a/cytoolz/dicttoolz.pxd b/cytoolz/dicttoolz.pxd index 3133dde..4b80e9b 100644 --- a/cytoolz/dicttoolz.pxd +++ b/cytoolz/dicttoolz.pxd @@ -38,7 +38,7 @@ cpdef object assoc(object d, object key, object value, object factory=*) cpdef object assoc_in(object d, object keys, object value, object factory=*) -cdef object c_dissoc(object d, object keys) +cdef object c_dissoc(object d, object keys, object factory=*) cpdef object update_in(object d, object keys, object func, object default=*, object factory=*) diff --git a/cytoolz/dicttoolz.pyx b/cytoolz/dicttoolz.pyx index dabf920..aa0c213 100644 --- a/cytoolz/dicttoolz.pyx +++ b/cytoolz/dicttoolz.pyx @@ -414,16 +414,24 @@ cpdef object assoc_in(object d, object keys, object value, object factory=dict): return rv -cdef object c_dissoc(object d, object keys): - cdef object rv, key - rv = copy(d) - for key in keys: - if key in rv: - del rv[key] +cdef object c_dissoc(object d, object keys, object factory=dict): + # implementation copied from toolz. Not benchmarked. + cdef object rv + rv = factory() + if len(keys) < len(d) * 0.6: + rv.update(d) + for key in keys: + if key in rv: + del rv[key] + else: + remaining = set(d) + remaining.difference_update(keys) + for k in remaining: + rv[k] = d[k] return rv -def dissoc(d, *keys): +def dissoc(d, *keys, **kwargs): """ Return a new dict with the given key(s) removed. @@ -437,7 +445,7 @@ def dissoc(d, *keys): >>> dissoc({'x': 1}, 'y') # Ignores missing keys {'x': 1} """ - return c_dissoc(d, keys) + return c_dissoc(d, keys, get_factory('dissoc', kwargs)) cpdef object update_in(object d, object keys, object func, object default=None, object factory=dict): diff --git a/cytoolz/functoolz.pxd b/cytoolz/functoolz.pxd index 6ee89f6..260cba9 100644 --- a/cytoolz/functoolz.pxd +++ b/cytoolz/functoolz.pxd @@ -38,6 +38,9 @@ cdef class Compose: cdef object c_compose(object funcs) +cdef object c_compose_left(object funcs) + + cdef object c_pipe(object data, object funcs) diff --git a/cytoolz/functoolz.pyx b/cytoolz/functoolz.pyx index 8e4baa5..f14aef8 100644 --- a/cytoolz/functoolz.pyx +++ b/cytoolz/functoolz.pyx @@ -3,8 +3,9 @@ import sys from functools import partial from operator import attrgetter from textwrap import dedent +from types import MethodType from cytoolz.utils import no_default -from cytoolz.compatibility import PY3, PY33, PY34, filter as ifilter, map as imap, reduce, import_module +from cytoolz.compatibility import PY3, PY34, filter as ifilter, map as imap, reduce, import_module import cytoolz._signatures as _sigs from toolz.functoolz import (InstanceProperty, instanceproperty, is_arity, @@ -21,15 +22,32 @@ from cpython.set cimport PyFrozenSet_New from cpython.tuple cimport PyTuple_Check, PyTuple_GET_SIZE -__all__ = ['identity', 'thread_first', 'thread_last', 'memoize', 'compose', +__all__ = ['identity', 'thread_first', 'thread_last', 'memoize', 'compose', 'compose_left', 'pipe', 'complement', 'juxt', 'do', 'curry', 'memoize', 'flip', - 'excepts'] + 'excepts', 'apply'] cpdef object identity(object x): return x +def apply(*func_and_args, **kwargs): + """ + Applies a function and returns the results + + >>> def double(x): return 2*x + >>> def inc(x): return x + 1 + >>> apply(double, 5) + 10 + + >>> tuple(map(apply, [double, inc, double], [10, 500, 8000])) + (20, 501, 16000) + """ + if not func_and_args: + raise TypeError('func argument is required') + return func_and_args[0](*func_and_args[1:], **kwargs) + + cdef object c_thread_first(object val, object forms): cdef object form, func cdef tuple args @@ -294,13 +312,7 @@ cdef class curry: property __signature__: def __get__(self): - try: - sig = inspect.signature(self.func) - except TypeError: - if PY33 and (getattr(self.func, '__module__') or '').startswith('cytoolz.'): - raise ValueError('callable %r is not supported by signature' % self.func) - raise - + sig = inspect.signature(self.func) args = self.args or () keywords = self.keywords or {} if is_partial_args(self.func, args, keywords, sig) is False: @@ -499,6 +511,41 @@ cdef class Compose: def __setstate__(self, state): self.funcs = state + def __repr__(self): + return '{.__class__.__name__}{!r}'.format( + self, tuple(reversed((self.first, ) + self.funcs))) + + def __eq__(self, other): + if isinstance(other, Compose): + return other.first == self.first and other.funcs == self.funcs + return NotImplemented + + def __ne__(self, other): + if isinstance(other, Compose): + return other.first != self.first or other.funcs != self.funcs + return NotImplemented + + def __hash__(self): + return hash(self.first) ^ hash(self.funcs) + + def __get__(self, obj, objtype): + if obj is None: + return self + elif PY3: + return MethodType(self, obj) + else: + return MethodType(self, obj, objtype) + + property __wrapped__: + def __get__(self): + return self.first + + property __signature__: + def __get__(self): + base = inspect.signature(self.first) + last = inspect.signature(self.funcs[-1]) + return base.replace(return_annotation=last.return_annotation) + property __name__: def __get__(self): try: @@ -554,11 +601,43 @@ def compose(*funcs): '4' See Also: + compose_left pipe """ return c_compose(funcs) +cdef object c_compose_left(object funcs): + if not funcs: + return identity + elif len(funcs) == 1: + return funcs[0] + else: + return Compose(*reversed(funcs)) + + +def compose_left(*funcs): + """ + Compose functions to operate in series. + + Returns a function that applies other functions in sequence. + + Functions are applied from left to right so that + ``compose_left(f, g, h)(x, y)`` is the same as ``h(g(f(x, y)))``. + + If no arguments are provided, the identity function (f(x) = x) is returned. + + >>> inc = lambda i: i + 1 + >>> compose_left(inc, str)(3) + '4' + + See Also: + compose + pipe + """ + return c_compose_left(funcs) + + cdef object c_pipe(object data, object funcs): cdef object func for func in funcs: @@ -583,6 +662,7 @@ def pipe(data, *funcs): See Also: compose + compose_left thread_first thread_last """ diff --git a/cytoolz/itertoolz.pxd b/cytoolz/itertoolz.pxd index 9684e44..57e0609 100644 --- a/cytoolz/itertoolz.pxd +++ b/cytoolz/itertoolz.pxd @@ -272,6 +272,9 @@ cpdef object topk(Py_ssize_t k, object seq, object key=*) cpdef object peek(object seq) +cpdef object peekn(Py_ssize_t n, object seq) + + cdef class random_sample: cdef object iter_seq cdef object prob diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index 9c4f7b1..bd3960a 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -24,7 +24,7 @@ __all__ = ['remove', 'accumulate', 'groupby', 'merge_sorted', 'interleave', 'first', 'second', 'nth', 'last', 'get', 'concat', 'concatv', 'mapcat', 'cons', 'interpose', 'frequencies', 'reduceby', 'iterate', 'sliding_window', 'partition', 'partition_all', 'count', 'pluck', - 'join', 'tail', 'diff', 'topk', 'peek', 'random_sample'] + 'join', 'tail', 'diff', 'topk', 'peek', 'peekn', 'random_sample'] cpdef object identity(object x): @@ -136,6 +136,8 @@ cpdef dict groupby(object key, object seq): 'M': [{'gender': 'M', 'name': 'Bob'}, {'gender': 'M', 'name': 'Charlie'}]} + Not to be confused with ``itertools.groupby`` + See Also: countby """ @@ -1252,6 +1254,8 @@ cpdef object join(object leftkey, object leftseq, This is a semi-streaming operation. The LEFT sequence is fully evaluated and placed into memory. The RIGHT sequence is evaluated lazily and so can be arbitrarily large. + (Note: If right_default is defined, then unique keys of rightseq + will also be stored in memory.) >>> friends = [('Alice', 'Edith'), ... ('Alice', 'Zhao'), @@ -1294,7 +1298,10 @@ cpdef object join(object leftkey, object leftseq, Usually the key arguments are callables to be applied to the sequences. If the keys are not obviously callable then it is assumed that indexing was - intended, e.g. the following is a legal change + intended, e.g. the following is a legal change. + The join is implemented as a hash join and the keys of leftseq must be + hashable. Additionally, if right_default is defined, then keys of rightseq + must also be hashable. >>> # result = join(second, friends, first, cities) >>> result = join(1, friends, 0, cities) # doctest: +SKIP @@ -1727,6 +1734,25 @@ cpdef object peek(object seq): return item, chain((item,), iterator) +cpdef object peekn(Py_ssize_t n, object seq): + """ + Retrieve the next n elements of a sequence + + Returns a tuple of the first n elements and an iterable equivalent + to the original, still having the elements retrieved. + + >>> seq = [0, 1, 2, 3, 4] + >>> first_two, seq = peekn(2, seq) + >>> first_two + (0, 1) + >>> list(seq) + [0, 1, 2, 3, 4] + """ + iterator = iter(seq) + peeked = tuple(take(n, iterator)) + return peeked, chain(iter(peeked), iterator) + + cdef class random_sample: """ random_sample(prob, seq, random_state=None) diff --git a/cytoolz/tests/test_curried.py b/cytoolz/tests/test_curried.py index a2ef1ad..af78ec6 100644 --- a/cytoolz/tests/test_curried.py +++ b/cytoolz/tests/test_curried.py @@ -2,8 +2,8 @@ import cytoolz.curried from cytoolz.curried import (take, first, second, sorted, merge_with, reduce, merge, operator as cop) -from cytoolz.compatibility import import_module from collections import defaultdict +from importlib import import_module from operator import add @@ -62,7 +62,7 @@ def test_curried_operator(): ) # Make sure this isn't totally empty. - assert len(set(vars(cop)) & set(['add', 'sub', 'mul'])) == 3 + assert len(set(vars(cop)) & {'add', 'sub', 'mul'}) == 3 def test_curried_namespace(): @@ -79,10 +79,10 @@ def should_curry(func): def curry_namespace(ns): - return dict( - (name, cytoolz.curry(f) if should_curry(f) else f) + return { + name: cytoolz.curry(f) if should_curry(f) else f for name, f in ns.items() if '__' not in name - ) + } from_cytoolz = curry_namespace(vars(cytoolz)) from_exceptions = curry_namespace(vars(exceptions)) diff --git a/cytoolz/tests/test_dicttoolz.py b/cytoolz/tests/test_dicttoolz.py index 6327631..e4de4df 100644 --- a/cytoolz/tests/test_dicttoolz.py +++ b/cytoolz/tests/test_dicttoolz.py @@ -90,16 +90,16 @@ def test_assoc(self): def test_dissoc(self): D, kw = self.D, self.kw - assert dissoc(D({"a": 1}), "a") == D({}) - assert dissoc(D({"a": 1, "b": 2}), "a") == D({"b": 2}) - assert dissoc(D({"a": 1, "b": 2}), "b") == D({"a": 1}) - assert dissoc(D({"a": 1, "b": 2}), "a", "b") == D({}) - assert dissoc(D({"a": 1}), "a") == dissoc(dissoc(D({"a": 1}), "a"), "a") + assert dissoc(D({"a": 1}), "a", **kw) == D({}) + assert dissoc(D({"a": 1, "b": 2}), "a", **kw) == D({"b": 2}) + assert dissoc(D({"a": 1, "b": 2}), "b", **kw) == D({"a": 1}) + assert dissoc(D({"a": 1, "b": 2}), "a", "b", **kw) == D({}) + assert dissoc(D({"a": 1}), "a", **kw) == dissoc(dissoc(D({"a": 1}), "a", **kw), "a", **kw) # Verify immutability: d = D({'x': 1}) oldd = d - d2 = dissoc(d, 'x') + d2 = dissoc(d, 'x', **kw) assert d is oldd assert d2 is not oldd diff --git a/cytoolz/tests/test_functoolz.py b/cytoolz/tests/test_functoolz.py index 1dd997a..9293025 100644 --- a/cytoolz/tests/test_functoolz.py +++ b/cytoolz/tests/test_functoolz.py @@ -1,7 +1,8 @@ -import platform - +import inspect from cytoolz.functoolz import (thread_first, thread_last, memoize, curry, - compose, pipe, complement, do, juxt, flip, excepts) + compose, compose_left, pipe, complement, do, juxt, + flip, excepts, apply) +from cytoolz.compatibility import PY3 from operator import add, mul, itemgetter from cytoolz.utils import raises from functools import partial @@ -23,6 +24,32 @@ def double(x): return 2 * x +class AlwaysEquals(object): + """useful to test correct __eq__ implementation of other objects""" + + def __eq__(self, other): + return True + + def __ne__(self, other): + return False + + +class NeverEquals(object): + """useful to test correct __eq__ implementation of other objects""" + + def __eq__(self, other): + return False + + def __ne__(self, other): + return True + + +def test_apply(): + assert apply(double, 5) == 10 + assert tuple(map(apply, [double, inc, double], [10, 500, 8000])) == (20, 501, 16000) + assert raises(TypeError, apply) + + def test_thread_first(): assert thread_first(2) == 2 assert thread_first(2, inc) == 3 @@ -201,12 +228,10 @@ def g(a=1, b=10, c=0): def h(x, func=int): return func(x) - if platform.python_implementation() != 'PyPy'\ - or platform.python_version_tuple()[0] != '3': # Bug on PyPy3<2.5 - # __init__ must not pick func as positional arg - assert curry(h)(0.0) == 0 - assert curry(h)(func=str)(0.0) == '0.0' - assert curry(h, func=str)(0.0) == '0.0' + # __init__ must not pick func as positional arg + assert curry(h)(0.0) == 0 + assert curry(h)(func=str)(0.0) == '0.0' + assert curry(h, func=str)(0.0) == '0.0' def test_curry_passes_errors(): @@ -328,7 +353,7 @@ def bar(a, b, c=1): b1 = curry(bar, 1, c=2) assert b1 != f1 - assert set([f1, f2, g1, h1, h2, h3, b1, b1()]) == set([f1, g1, h1, b1]) + assert {f1, f2, g1, h1, h2, h3, b1, b1()} == {f1, g1, h1, b1} # test unhashable input unhash1 = curry(foo, []) @@ -499,17 +524,54 @@ def _should_curry(self, args, kwargs, exc=None): """ -def test_compose(): - assert compose()(0) == 0 - assert compose(inc)(0) == 1 - assert compose(double, inc)(0) == 2 - assert compose(str, iseven, inc, double)(3) == "False" - assert compose(str, add)(1, 2) == '3' +def generate_compose_test_cases(): + """ + Generate test cases for parametrized tests of the compose function. + """ - def f(a, b, c=10): + def add_then_multiply(a, b, c=10): return (a + b) * c - assert compose(str, inc, f)(1, 2, c=3) == '10' + return ( + ( + (), # arguments to compose() + (0,), {}, # positional and keyword args to the Composed object + 0 # expected result + ), + ( + (inc,), + (0,), {}, + 1 + ), + ( + (double, inc), + (0,), {}, + 2 + ), + ( + (str, iseven, inc, double), + (3,), {}, + "False" + ), + ( + (str, add), + (1, 2), {}, + '3' + ), + ( + (str, inc, add_then_multiply), + (1, 2), {"c": 3}, + '10' + ), + ) + + +def test_compose(): + for (compose_args, args, kw, expected) in generate_compose_test_cases(): + assert compose(*compose_args)(*args, **kw) == expected + + +def test_compose_metadata(): # Define two functions with different names def f(a): @@ -529,10 +591,94 @@ def g(a): assert composed.__name__ == 'Compose' assert composed.__doc__ == 'A composition of functions' + # print(repr(composed)) + # print('Compose({!r}, {!r})'.format(f, h)) + assert repr(composed) == 'Compose({!r}, {!r})'.format(f, h) + + assert composed == compose(f, h) + assert composed == AlwaysEquals() + assert not composed == compose(h, f) + assert not composed == object() + assert not composed == NeverEquals() -def test_compose_module(): - # regression test for #103 - assert compose().__module__ == 'cytoolz.functoolz' + assert composed != compose(h, f) + assert composed != NeverEquals() + assert composed != object() + assert not composed != compose(f, h) + assert not composed != AlwaysEquals() + + assert hash(composed) == hash(compose(f, h)) + assert hash(composed) != hash(compose(h, f)) + + bindable = compose(str, lambda x: x*2, lambda x, y=0: int(x) + y) + + class MyClass: + + def __int__(self): + return 8 + + my_method = bindable + my_static_method = staticmethod(bindable) + + assert MyClass.my_method(3) == '6' + assert MyClass.my_method(3, y=2) == '10' + assert MyClass.my_static_method(2) == '4' + assert MyClass().my_method() == '16' + assert MyClass().my_method(y=3) == '22' + assert MyClass().my_static_method(0) == '0' + assert MyClass().my_static_method(0, 1) == '2' + + assert compose(f, h).__wrapped__ is h + # assert compose(f, h).__class__.__wrapped__ is None + + # __signature__ is python3 only + if PY3: + + def myfunc(a, b, c, *d, **e): + return 4 + + def otherfunc(f): + return 'result: {}'.format(f) + + # set annotations compatibly with python2 syntax + myfunc.__annotations__ = { + 'a': int, + 'b': str, + 'c': float, + 'd': int, + 'e': bool, + 'return': int, + } + otherfunc.__annotations__ = {'f': int, 'return': str} + + composed = compose(otherfunc, myfunc) + sig = inspect.signature(composed) + assert sig.parameters == inspect.signature(myfunc).parameters + assert sig.return_annotation == str + + class MyClass: + method = composed + + assert len(inspect.signature(MyClass().method).parameters) == 4 + + +def generate_compose_left_test_cases(): + """ + Generate test cases for parametrized tests of the compose function. + + These are based on, and equivalent to, those produced by + enerate_compose_test_cases(). + """ + return tuple( + (tuple(reversed(compose_args)), args, kwargs, expected) + for (compose_args, args, kwargs, expected) + in generate_compose_test_cases() + ) + + +def test_compose_left(): + for (compose_left_args, args, kw, expected) in generate_compose_left_test_cases(): + assert compose_left(*compose_left_args)(*args, **kw) == expected def test_pipe(): @@ -650,4 +796,3 @@ def raise_(a): excepting = excepts(object(), object(), object()) assert excepting.__name__ == 'excepting' assert excepting.__doc__ == excepts.__doc__ - diff --git a/cytoolz/tests/test_inspect_args.py b/cytoolz/tests/test_inspect_args.py index 17b64ad..bbe68cb 100644 --- a/cytoolz/tests/test_inspect_args.py +++ b/cytoolz/tests/test_inspect_args.py @@ -7,7 +7,7 @@ num_required_args, has_varargs, has_keywords) from cytoolz._signatures import builtins import cytoolz._signatures as _sigs -from cytoolz.compatibility import PY3, PY33 +from cytoolz.compatibility import PY3 from cytoolz.utils import raises @@ -437,7 +437,7 @@ def is_missing(modname, name, func): if missing: messages = [] for modname, names in sorted(missing.items()): - msg = '{0}:\n {1}'.format(modname, '\n '.join(sorted(names))) + msg = '{}:\n {}'.format(modname, '\n '.join(sorted(names))) messages.append(msg) message = 'Missing introspection for the following callables:\n\n' raise AssertionError(message + '\n\n'.join(messages)) @@ -495,6 +495,6 @@ def __wrapped__(self): if PY3: assert inspect.signature(func) == inspect.signature(wrapped) - assert num_required_args(Wrapped) == (False if PY33 else None) + assert num_required_args(Wrapped) is None _sigs.signatures[Wrapped] = (_sigs.expand_sig((0, lambda func: None)),) assert num_required_args(Wrapped) == 1 diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index c7d99b8..649e879 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -13,7 +13,7 @@ reduceby, iterate, accumulate, sliding_window, count, partition, partition_all, take_nth, pluck, join, - diff, topk, peek, random_sample) + diff, topk, peek, peekn, random_sample) from cytoolz.compatibility import range, filter from operator import add, mul @@ -271,7 +271,7 @@ def set_add(s, i): return s assert reduceby(iseven, set_add, [1, 2, 3, 4, 1, 2], set) == \ - {True: set([2, 4]), False: set([1, 3])} + {True: {2, 4}, False: {1, 3}} def test_iterate(): @@ -289,6 +289,7 @@ def binop(a, b): start = object() assert list(accumulate(binop, [], start)) == [start] + assert list(accumulate(binop, [])) == [] assert list(accumulate(add, [1, 2, 3], no_default2)) == [1, 3, 6] @@ -318,6 +319,17 @@ def test_partition_all(): assert list(partition_all(3, range(5))) == [(0, 1, 2), (3, 4)] assert list(partition_all(2, [])) == [] + # Regression test: https://github.com/pycytoolz/cytoolz/issues/387 + class NoCompare(object): + def __eq__(self, other): + if self.__class__ == other.__class__: + return True + raise ValueError() + obj = NoCompare() + result = [(obj, obj, obj, obj), (obj, obj, obj)] + assert list(partition_all(4, [obj]*7)) == result + assert list(partition_all(4, iter([obj]*7))) == result + def test_count(): assert count((1, 2, 3)) == 3 @@ -356,10 +368,10 @@ def addpair(pair): result = set(starmap(add, join(first, names, second, fruit))) - expected = set([((1, 'one', 'apple', 1)), - ((1, 'one', 'orange', 1)), - ((2, 'two', 'banana', 2)), - ((2, 'two', 'coconut', 2))]) + expected = {(1, 'one', 'apple', 1), + (1, 'one', 'orange', 1), + (2, 'two', 'banana', 2), + (2, 'two', 'coconut', 2)} assert result == expected @@ -397,14 +409,14 @@ def test_join_double_repeats(): result = set(starmap(add, join(first, names, second, fruit))) - expected = set([((1, 'one', 'apple', 1)), - ((1, 'one', 'orange', 1)), - ((2, 'two', 'banana', 2)), - ((2, 'two', 'coconut', 2)), - ((1, 'uno', 'apple', 1)), - ((1, 'uno', 'orange', 1)), - ((2, 'dos', 'banana', 2)), - ((2, 'dos', 'coconut', 2))]) + expected = {(1, 'one', 'apple', 1), + (1, 'one', 'orange', 1), + (2, 'two', 'banana', 2), + (2, 'two', 'coconut', 2), + (1, 'uno', 'apple', 1), + (1, 'uno', 'orange', 1), + (2, 'dos', 'banana', 2), + (2, 'dos', 'coconut', 2)} assert result == expected @@ -415,21 +427,21 @@ def test_join_missing_element(): result = set(starmap(add, join(first, names, second, fruit))) - expected = set([((1, 'one', 'orange', 1))]) + expected = {(1, 'one', 'orange', 1)} assert result == expected def test_left_outer_join(): result = set(join(identity, [1, 2], identity, [2, 3], left_default=None)) - expected = set([(2, 2), (None, 3)]) + expected = {(2, 2), (None, 3)} assert result == expected def test_right_outer_join(): result = set(join(identity, [1, 2], identity, [2, 3], right_default=None)) - expected = set([(2, 2), (1, None)]) + expected = {(2, 2), (1, None)} assert result == expected @@ -437,7 +449,7 @@ def test_right_outer_join(): def test_outer_join(): result = set(join(identity, [1, 2], identity, [2, 3], left_default=None, right_default=None)) - expected = set([(2, 2), (1, None), (None, 3)]) + expected = {(2, 2), (1, None), (None, 3)} assert result == expected @@ -496,12 +508,23 @@ def test_topk_is_stable(): def test_peek(): alist = ["Alice", "Bob", "Carol"] element, blist = peek(alist) - element == alist[0] + assert element == alist[0] assert list(blist) == alist assert raises(StopIteration, lambda: peek([])) +def test_peekn(): + alist = ("Alice", "Bob", "Carol") + elements, blist = peekn(2, alist) + assert elements == alist[:2] + assert tuple(blist) == alist + + elements, blist = peekn(len(alist) * 4, alist) + assert elements == alist + assert tuple(blist) == alist + + def test_random_sample(): alist = list(range(100)) diff --git a/cytoolz/tests/test_none_safe.py b/cytoolz/tests/test_none_safe.py index 8dbd7cc..7d00a93 100644 --- a/cytoolz/tests/test_none_safe.py +++ b/cytoolz/tests/test_none_safe.py @@ -118,6 +118,10 @@ def test_functoolz(): assert raises(TypeError, lambda: compose(None, None)()) tested.append('compose') + assert compose_left(None) is None + assert raises(TypeError, lambda: compose_left(None, None)()) + tested.append('compose_left') + assert raises(TypeError, lambda: curry(None)) tested.append('curry') @@ -147,6 +151,10 @@ def test_functoolz(): assert flip(lambda a, b: (a, b))(None)(None) == (None, None) tested.append('flip') + assert apply(identity, None) is None + assert raises(TypeError, lambda: apply(None)) + tested.append('apply') + excepts(None, lambda x: x) excepts(TypeError, None) tested.append('excepts') @@ -302,6 +310,10 @@ def test_itertoolz(): assert raises(TypeError, lambda: peek(None)) tested.append('peek') + assert raises(TypeError, lambda: peekn(None, [1, 2, 3])) + assert raises(TypeError, lambda: peekn(3, None)) + tested.append('peekn') + assert raises(TypeError, lambda: list(random_sample(None, [1]))) assert raises(TypeError, lambda: list(random_sample(0.1, None))) tested.append('random_sample') diff --git a/cytoolz/tests/test_serialization.py b/cytoolz/tests/test_serialization.py index 7f95cbe..645994a 100644 --- a/cytoolz/tests/test_serialization.py +++ b/cytoolz/tests/test_serialization.py @@ -2,7 +2,7 @@ import cytoolz import cytoolz.curried import pickle -from cytoolz.compatibility import PY3, PY33, PY34 +from cytoolz.compatibility import PY3 from cytoolz.utils import raises @@ -81,7 +81,7 @@ def g1(self): def __reduce__(self): """Allow us to serialize instances of GlobalCurried""" - return (GlobalCurried, (self.x, self.y)) + return GlobalCurried, (self.x, self.y) @cytoolz.curry class NestedCurried(object): @@ -98,7 +98,7 @@ def g2(self): def __reduce__(self): """Allow us to serialize instances of NestedCurried""" - return (GlobalCurried.NestedCurried, (self.x, self.y)) + return GlobalCurried.NestedCurried, (self.x, self.y) class Nested(object): def __init__(self, x, y): @@ -148,7 +148,7 @@ def preserves_identity(obj): # If we add `curry.__getattr__` forwarding, the following tests will pass - # if not PY33 and not PY34: + # if not PY34: # assert preserves_identity(GlobalCurried.func.g1) # assert preserves_identity(GlobalCurried.func.NestedCurried.func.g2) # assert preserves_identity(GlobalCurried.func.Nested) @@ -159,7 +159,7 @@ def preserves_identity(obj): # assert preserves_identity(GlobalCurried.NestedCurried) # assert preserves_identity(GlobalCurried.NestedCurried.f2) # assert preserves_identity(GlobalCurried.Nested.f3) - # if not PY33 and not PY34: + # if not PY34: # assert preserves_identity(GlobalCurried.g1) # assert preserves_identity(GlobalCurried.NestedCurried.g2) # assert preserves_identity(GlobalCurried.Nested) @@ -175,7 +175,7 @@ def preserves_identity(obj): # assert func1 is not func2 # assert func1(4) == func2(4) == 10 # - # if not PY33 and not PY34: + # if not PY34: # nested3 = GlobalCurried.func.Nested(1, 2) # nested4 = pickle.loads(pickle.dumps(nested3)) # assert nested3 is not nested4 From d1acb280f30612316eea5ca95bc42f3fd9d8c3f7 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 9 Jul 2019 16:29:55 -0500 Subject: [PATCH 134/146] update signatures --- cytoolz/_signatures.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cytoolz/_signatures.py b/cytoolz/_signatures.py index cc5a3c4..0864973 100644 --- a/cytoolz/_signatures.py +++ b/cytoolz/_signatures.py @@ -10,7 +10,7 @@ assoc_in=[ lambda d, keys, value, factory=dict: None], dissoc=[ - lambda d, *keys: None], + lambda d, *keys, **kwargs: None], get_in=[ lambda keys, coll, default=None, no_default=False: None], itemfilter=[ @@ -34,12 +34,16 @@ ) cytoolz_info['cytoolz.functoolz'] = dict( + apply=[ + lambda func, *args, **kwargs: None], Compose=[ lambda *funcs: None], complement=[ lambda func: None], compose=[ lambda *funcs: None], + compose_left=[ + lambda *funcs: None], curry=[ lambda *args, **kwargs: None], do=[ @@ -125,6 +129,8 @@ lambda n, seq: None], peek=[ lambda seq: None], + peekn=[ + lambda n, seq: None], pluck=[ lambda ind, seqs, default=None: None], random_sample=[ From 162471e5e4bb0145505f436a9fcba458c2080cc5 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 9 Jul 2019 17:05:27 -0500 Subject: [PATCH 135/146] add `apply` to `curried` --- cytoolz/_signatures.py | 2 +- cytoolz/curried/__init__.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cytoolz/_signatures.py b/cytoolz/_signatures.py index 0864973..c265fdb 100644 --- a/cytoolz/_signatures.py +++ b/cytoolz/_signatures.py @@ -35,7 +35,7 @@ cytoolz_info['cytoolz.functoolz'] = dict( apply=[ - lambda func, *args, **kwargs: None], + lambda *func_and_args, **kwargs], Compose=[ lambda *funcs: None], complement=[ diff --git a/cytoolz/curried/__init__.py b/cytoolz/curried/__init__.py index 281f62f..ad2c7fb 100644 --- a/cytoolz/curried/__init__.py +++ b/cytoolz/curried/__init__.py @@ -57,6 +57,7 @@ from .exceptions import merge, merge_with accumulate = cytoolz.curry(cytoolz.accumulate) +apply - cytoolz.curry(cytoolz.apply) assoc = cytoolz.curry(cytoolz.assoc) assoc_in = cytoolz.curry(cytoolz.assoc_in) cons = cytoolz.curry(cytoolz.cons) From 676f75425620bd6eaaf19323ad74d851f2c1c3d7 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 9 Jul 2019 19:50:32 -0500 Subject: [PATCH 136/146] Oups --- cytoolz/_signatures.py | 2 +- cytoolz/curried/__init__.py | 1 - cytoolz/tests/test_functoolz.py | 6 +++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cytoolz/_signatures.py b/cytoolz/_signatures.py index c265fdb..aded849 100644 --- a/cytoolz/_signatures.py +++ b/cytoolz/_signatures.py @@ -35,7 +35,7 @@ cytoolz_info['cytoolz.functoolz'] = dict( apply=[ - lambda *func_and_args, **kwargs], + lambda *func_and_args, **kwargs: None], Compose=[ lambda *funcs: None], complement=[ diff --git a/cytoolz/curried/__init__.py b/cytoolz/curried/__init__.py index ad2c7fb..281f62f 100644 --- a/cytoolz/curried/__init__.py +++ b/cytoolz/curried/__init__.py @@ -57,7 +57,6 @@ from .exceptions import merge, merge_with accumulate = cytoolz.curry(cytoolz.accumulate) -apply - cytoolz.curry(cytoolz.apply) assoc = cytoolz.curry(cytoolz.assoc) assoc_in = cytoolz.curry(cytoolz.assoc_in) cons = cytoolz.curry(cytoolz.cons) diff --git a/cytoolz/tests/test_functoolz.py b/cytoolz/tests/test_functoolz.py index 9293025..0eb520d 100644 --- a/cytoolz/tests/test_functoolz.py +++ b/cytoolz/tests/test_functoolz.py @@ -1,4 +1,5 @@ import inspect +import cytoolz from cytoolz.functoolz import (thread_first, thread_last, memoize, curry, compose, compose_left, pipe, complement, do, juxt, flip, excepts, apply) @@ -591,8 +592,6 @@ def g(a): assert composed.__name__ == 'Compose' assert composed.__doc__ == 'A composition of functions' - # print(repr(composed)) - # print('Compose({!r}, {!r})'.format(f, h)) assert repr(composed) == 'Compose({!r}, {!r})'.format(f, h) assert composed == compose(f, h) @@ -629,7 +628,8 @@ def __int__(self): assert MyClass().my_static_method(0, 1) == '2' assert compose(f, h).__wrapped__ is h - # assert compose(f, h).__class__.__wrapped__ is None + if hasattr(cytoolz, 'sandbox'): # only test this with Python version (i.e., not Cython) + assert compose(f, h).__class__.__wrapped__ is None # __signature__ is python3 only if PY3: From 41429499248bd7d7702664c540f43f5454eaf6ec Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 9 Jul 2019 19:55:31 -0500 Subject: [PATCH 137/146] clean up dissoc in curried --- cytoolz/curried/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cytoolz/curried/__init__.py b/cytoolz/curried/__init__.py index 281f62f..87fa7f2 100644 --- a/cytoolz/curried/__init__.py +++ b/cytoolz/curried/__init__.py @@ -36,7 +36,6 @@ count, curry, diff, - dissoc, first, flip, frequencies, From 24614d78f45cab68b8b47b2f38f46d822ddb6c5e Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 9 Jul 2019 20:40:57 -0500 Subject: [PATCH 138/146] Update setup.py Python version classifiers --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 1a5f588..28e4a0e 100644 --- a/setup.py +++ b/setup.py @@ -99,13 +99,12 @@ 'Programming Language :: Cython', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Scientific/Engineering', 'Topic :: Scientific/Engineering :: Information Analysis', 'Topic :: Software Development', From e90e13f86043f18f474a52cef0eaeef136d03568 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 10 Jul 2019 11:38:11 -0500 Subject: [PATCH 139/146] Fix #123 when sliding_window n was larger than the sequence --- cytoolz/__init__.py | 4 ++-- cytoolz/itertoolz.pyx | 10 +++++----- cytoolz/tests/test_inspect_args.py | 1 - cytoolz/tests/test_itertoolz.py | 1 + 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cytoolz/__init__.py b/cytoolz/__init__.py index 01746e9..81f34a3 100644 --- a/cytoolz/__init__.py +++ b/cytoolz/__init__.py @@ -8,8 +8,6 @@ from .compatibility import map, filter -# from . import sandbox - from functools import partial, reduce sorted = sorted @@ -21,6 +19,8 @@ flip = functoolz.flip = curry(functoolz.flip) memoize = functoolz.memoize = curry(functoolz.memoize) +from . import curried # sandbox + functoolz._sigs.update_signature_registry() from ._version import __version__, __toolz_version__ diff --git a/cytoolz/itertoolz.pyx b/cytoolz/itertoolz.pyx index bd3960a..29193b4 100644 --- a/cytoolz/itertoolz.pyx +++ b/cytoolz/itertoolz.pyx @@ -964,8 +964,7 @@ cdef class sliding_window: cdef Py_ssize_t i self.iterseq = iter(seq) self.prev = PyTuple_New(n) - for i in range(1, n): - seq = next(self.iterseq) + for i, seq in enumerate(islice(self.iterseq, n-1), 1): Py_INCREF(seq) PyTuple_SET_ITEM(self.prev, i, seq) self.n = n @@ -977,14 +976,15 @@ cdef class sliding_window: cdef tuple current cdef object item cdef Py_ssize_t i + + item = next(self.iterseq) current = PyTuple_New(self.n) + Py_INCREF(item) + PyTuple_SET_ITEM(current, self.n-1, item) for i in range(1, self.n): item = self.prev[i] Py_INCREF(item) PyTuple_SET_ITEM(current, i-1, item) - item = next(self.iterseq) - Py_INCREF(item) - PyTuple_SET_ITEM(current, self.n-1, item) self.prev = current return current diff --git a/cytoolz/tests/test_inspect_args.py b/cytoolz/tests/test_inspect_args.py index bbe68cb..d39278e 100644 --- a/cytoolz/tests/test_inspect_args.py +++ b/cytoolz/tests/test_inspect_args.py @@ -402,7 +402,6 @@ def add_blacklist(mod, attr): blacklist.add(getattr(mod, attr)) add_blacklist(builtins, 'basestring') - add_blacklist(builtins, 'breakpoint') add_blacklist(builtins, 'NoneType') add_blacklist(builtins, '__metaclass__') add_blacklist(builtins, 'sequenceiterator') diff --git a/cytoolz/tests/test_itertoolz.py b/cytoolz/tests/test_itertoolz.py index 649e879..86c4370 100644 --- a/cytoolz/tests/test_itertoolz.py +++ b/cytoolz/tests/test_itertoolz.py @@ -304,6 +304,7 @@ def test_sliding_window(): def test_sliding_window_of_short_iterator(): assert list(sliding_window(3, [1, 2])) == [] + assert list(sliding_window(7, [1, 2])) == [] def test_partition(): From b66732f7f51937e85f5112481baf9db9c97b2ad2 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 11 Jul 2019 13:47:02 -0500 Subject: [PATCH 140/146] Safely iterate over non-dict mappings. See #127 --- cytoolz/dicttoolz.pxd | 4 ++++ cytoolz/dicttoolz.pyx | 20 +++++++++++++++++--- cytoolz/tests/test_dicttoolz.py | 9 +++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/cytoolz/dicttoolz.pxd b/cytoolz/dicttoolz.pxd index 4b80e9b..22d91bc 100644 --- a/cytoolz/dicttoolz.pxd +++ b/cytoolz/dicttoolz.pxd @@ -1,6 +1,10 @@ from cpython.ref cimport PyObject # utility functions to perform iteration over dicts or generic mapping +cdef class _iter_mapping: + cdef object it + cdef object cur + ctypedef int (*f_map_next)(object p, Py_ssize_t *ppos, PyObject* *pkey, PyObject* *pval) except -1 cdef f_map_next get_map_iter(object d, PyObject* *ptr) except NULL diff --git a/cytoolz/dicttoolz.pyx b/cytoolz/dicttoolz.pyx index aa0c213..8235899 100644 --- a/cytoolz/dicttoolz.pyx +++ b/cytoolz/dicttoolz.pyx @@ -16,6 +16,20 @@ __all__ = ['merge', 'merge_with', 'valmap', 'keymap', 'itemmap', 'valfilter', 'update_in'] +cdef class _iter_mapping: + """ Keep a handle on the current item to prevent memory clean up too early""" + def __cinit__(self, object it): + self.it = it + self.cur = None + + def __iter__(self): + return self + + def __next__(self): + self.cur = next(self.it) + return self.cur + + cdef int PyMapping_Next(object p, Py_ssize_t *ppos, PyObject* *pkey, PyObject* *pval) except -1: """Mimic "PyDict_Next" interface, but for any mapping""" cdef PyObject *obj @@ -24,7 +38,7 @@ cdef int PyMapping_Next(object p, Py_ssize_t *ppos, PyObject* *pkey, PyObject* * return 0 pkey[0] = (obj)[0] pval[0] = (obj)[1] - Py_XDECREF(obj) + Py_XDECREF(obj) # removing this results in memory leak return 1 @@ -53,10 +67,10 @@ cdef f_map_next get_map_iter(object d, PyObject* *ptr) except NULL: val = d rv = &PyDict_Next_Compat elif hasattr(d, 'iteritems'): - val = iter(d.iteritems()) + val = _iter_mapping(iter(d.iteritems())) rv = &PyMapping_Next else: - val = iter(d.items()) + val = _iter_mapping(iter(d.items())) rv = &PyMapping_Next Py_INCREF(val) ptr[0] = val diff --git a/cytoolz/tests/test_dicttoolz.py b/cytoolz/tests/test_dicttoolz.py index e4de4df..78ca675 100644 --- a/cytoolz/tests/test_dicttoolz.py +++ b/cytoolz/tests/test_dicttoolz.py @@ -1,7 +1,9 @@ from collections import defaultdict as _defaultdict +import os from cytoolz.dicttoolz import (merge, merge_with, valmap, keymap, update_in, assoc, dissoc, keyfilter, valfilter, itemmap, itemfilter, assoc_in) +from cytoolz.functoolz import identity from cytoolz.utils import raises from cytoolz.compatibility import PY3 @@ -250,3 +252,10 @@ class TestCustomMapping(TestDict): """ D = CustomMapping kw = {'factory': lambda: CustomMapping()} + + +def test_environ(): + # See: https://github.com/pytoolz/cytoolz/issues/127 + assert keymap(identity, os.environ) == os.environ + assert valmap(identity, os.environ) == os.environ + assert itemmap(identity, os.environ) == os.environ From da99d69afa2caeb7ac0213d2e71cfd55dd20f63f Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 11 Jul 2019 14:37:01 -0500 Subject: [PATCH 141/146] Update readme to reflect minimum Python version --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 703cb8a..4ece83b 100644 --- a/README.rst +++ b/README.rst @@ -48,7 +48,7 @@ Install Dependencies ------------ -``cytoolz`` supports Python 2.6+ and Python 3.3+ with a common codebase. +``cytoolz`` supports Python 2.7+ and Python 3.4+ with a common codebase. It is developed in Cython, but requires no dependecies other than CPython and a C compiler. Like ``toolz``, it is a light weight dependency. From d8852c82408a64e44171831e38ad11f9c856a429 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 11 Jul 2019 17:24:43 -0500 Subject: [PATCH 142/146] Just released! Bump to next dev version: 0.10.1dev --- conda.recipe/meta.yaml | 2 +- cytoolz/_version.py | 4 ++-- cytoolz/tests/test_dicttoolz.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 861d82b..87fbb3b 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: cytoolz - version: "0.9.0" + version: "0.10.0" build: number: {{environ.get('BINSTAR_BUILD', 1)}} diff --git a/cytoolz/_version.py b/cytoolz/_version.py index c0b02e7..7855ccf 100644 --- a/cytoolz/_version.py +++ b/cytoolz/_version.py @@ -1,2 +1,2 @@ -__version__ = '0.9.1dev' -__toolz_version__ = '0.9.0' +__version__ = '0.10.1dev' +__toolz_version__ = '0.10.0' diff --git a/cytoolz/tests/test_dicttoolz.py b/cytoolz/tests/test_dicttoolz.py index 78ca675..0a4cd27 100644 --- a/cytoolz/tests/test_dicttoolz.py +++ b/cytoolz/tests/test_dicttoolz.py @@ -255,7 +255,7 @@ class TestCustomMapping(TestDict): def test_environ(): - # See: https://github.com/pytoolz/cytoolz/issues/127 + # See: https://github.com/pycytoolz/cycytoolz/issues/127 assert keymap(identity, os.environ) == os.environ assert valmap(identity, os.environ) == os.environ assert itemmap(identity, os.environ) == os.environ From f9d4019189b302b8ff5d0a7702f92a3c7ff19225 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 3 Nov 2019 08:27:07 -0600 Subject: [PATCH 143/146] Test Python 3.8 (#136) and use Cython if available (#134) Python 3.8 failed to install cytoolz b/c the .c files created by Cython were not forward-compatible. This justifies always using Cython if available. Also, add Cython as an optional dependency via `extras_require` in setup.py. This should allow `pip install "cytoolz[cython]"` to do the right thing. --- .travis.yml | 10 ++-------- setup.py | 23 ++++++++++++++--------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 40a1b37..196e00e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,14 +4,8 @@ python: - "3.4" - "3.5" - "3.6" - - "3.7-dev" - -# Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs -matrix: - include: - - python: 3.7 - dist: xenial - sudo: true + - "3.7" + - "3.8" before_install: - pip install git+https://github.com/pytoolz/toolz.git diff --git a/setup.py b/setup.py index 28e4a0e..fcdc216 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,11 @@ """ Build ``cytoolz`` with or without Cython. -Deployed versions of CyToolz do not rely on Cython by default even if the -user has Cython installed. A C compiler is used to compile the distributed -*.c files instead. +By default, CyToolz will be built using Cython if available. +If Cython is not available, then the default C compiler will be used +to compile the distributed *.c files instead. -Pass "--cython" or "--with-cython" as a command line argument to setup.py -to build the project using Cython. +Pass "--cython" or "--with-cython" as a command line argument to setup.py to +force the project to build using Cython (and fail if Cython is unavailable). Pass "--no-cython" or "--without-cython" to disable usage of Cython. @@ -29,8 +29,8 @@ except ImportError: has_cython = False -is_dev = 'dev' in VERSION -use_cython = is_dev or '--cython' in sys.argv or '--with-cython' in sys.argv +use_cython = True +force_cython = 'dev' in VERSION if '--no-cython' in sys.argv: use_cython = False sys.argv.remove('--no-cython') @@ -38,14 +38,16 @@ use_cython = False sys.argv.remove('--without-cython') if '--cython' in sys.argv: + force_cython = True sys.argv.remove('--cython') if '--with-cython' in sys.argv: + force_cython = True sys.argv.remove('--with-cython') if use_cython and not has_cython: - if is_dev: + if force_cython: raise RuntimeError('Cython required to build dev version of cytoolz.') - print('WARNING: Cython not installed. Building without Cython.') + print('ALERT: Cython not installed. Building without Cython.') use_cython = False if use_cython: @@ -67,6 +69,7 @@ from Cython.Compiler.Options import directive_defaults directive_defaults['embedsignature'] = True directive_defaults['binding'] = True + directive_defaults['language_level'] = 2 # TODO: drop Python 2.7 and update this (and code) to 3 ext_modules = cythonize(ext_modules) setup( @@ -105,6 +108,7 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Topic :: Scientific/Engineering', 'Topic :: Scientific/Engineering :: Information Analysis', 'Topic :: Software Development', @@ -113,5 +117,6 @@ 'Topic :: Utilities', ], install_requires=['toolz >= 0.8.0'], + extras_require={'cython': ['cython']}, zip_safe=False, ) From b1b57d28ba007eda9dd7cc4db7b859490be11f5b Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 3 Nov 2019 15:23:49 -0600 Subject: [PATCH 144/146] Cythonize everything if we are using Cython with a non-dev version. --- setup.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index fcdc216..fcaef43 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,8 @@ has_cython = False use_cython = True -force_cython = 'dev' in VERSION +is_dev = 'dev' in VERSION +strict_cython = is_dev if '--no-cython' in sys.argv: use_cython = False sys.argv.remove('--no-cython') @@ -38,14 +39,14 @@ use_cython = False sys.argv.remove('--without-cython') if '--cython' in sys.argv: - force_cython = True + strict_cython = True sys.argv.remove('--cython') if '--with-cython' in sys.argv: - force_cython = True + strict_cython = True sys.argv.remove('--with-cython') if use_cython and not has_cython: - if force_cython: + if strict_cython: raise RuntimeError('Cython required to build dev version of cytoolz.') print('ALERT: Cython not installed. Building without Cython.') use_cython = False @@ -70,7 +71,9 @@ directive_defaults['embedsignature'] = True directive_defaults['binding'] = True directive_defaults['language_level'] = 2 # TODO: drop Python 2.7 and update this (and code) to 3 - ext_modules = cythonize(ext_modules) + # The distributed *.c files may not be forward compatible. + # If we are cythonizing a non-dev version, then force everything to cythonize. + ext_modules = cythonize(ext_modules, force=not is_dev) setup( name='cytoolz', From 5a8a2718ca6364c28923c71728cd7c220c2fc736 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 3 Nov 2019 15:44:08 -0600 Subject: [PATCH 145/146] Update docs with a good explanation of when we cythonize or use `*.c` files --- setup.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/setup.py b/setup.py index fcaef43..d7fd68d 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,17 @@ For convenience, developmental versions (with 'dev' in the version number) automatically use Cython unless disabled via a command line argument. +To summarize differently, the rules are as follows (apply first applicable rule): + + 1. If `--no-cython` or `--without-cython` are used, then only build from `.*c` files. + 2. If this is a dev version, then cythonize only the files that have changed. + 3. If `--cython` or `--with-cython` are used, then force cythonize all files. + 4. If no arguments are passed, then force cythonize all files if Cython is available, + else build from `*.c` files. This is default when installing via pip. + +By forcing cythonization of all files (except in dev) if Cython is available, +we avoid the case where the generated `*.c` files are not forward-compatible. + """ import os.path import sys From e6d62e55803b51cd63a18a00221812a5c5bb1734 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Sat, 21 Dec 2019 07:38:25 +1100 Subject: [PATCH 146/146] Fix simple typo: occurence -> occurrence There is a small typo in cytoolz/tests/test_none_safe.py. Should read `occurrence` rather than `occurence`. --- cytoolz/tests/test_none_safe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cytoolz/tests/test_none_safe.py b/cytoolz/tests/test_none_safe.py index 7d00a93..63f45dc 100644 --- a/cytoolz/tests/test_none_safe.py +++ b/cytoolz/tests/test_none_safe.py @@ -4,7 +4,7 @@ Python's C API that expect a specific type but receive None instead can cause problems such as throwing an uncatchable SystemError (and some systems may segfault instead). We obviously don't what that to happen! As the tests -below discovered, this turned out to be a rare occurence. The only changes +below discovered, this turned out to be a rare occurrence. The only changes required were to use `d.copy()` instead of `PyDict_Copy(d)`, and to always return Python objects from functions instead of int or bint (so exceptions can propagate).