From e1b9ba0f22c03da31b59ac6cecbefc1eb58e4752 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Sat, 19 Jan 2019 21:36:00 +1100 Subject: [PATCH 01/32] add docstring for Model namedtuple --- gensim/models/_fasttext_bin.py | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/gensim/models/_fasttext_bin.py b/gensim/models/_fasttext_bin.py index 31d85c5074..c5ac212c4e 100644 --- a/gensim/models/_fasttext_bin.py +++ b/gensim/models/_fasttext_bin.py @@ -79,6 +79,49 @@ def _yield_field_names(): _FIELD_NAMES = sorted(set(_yield_field_names())) Model = collections.namedtuple('Model', _FIELD_NAMES) +"""Holds data loaded from the Facebook binary. + +Fields +------ +dim : int + The dimensionality of the vectors. +ws : int + The window size. +epoch : int + The number of training epochs. +neg : int + If non-zero, indicates that the model uses negative sampling. +loss : int + If equal to 1, indicates that the model uses hierarchical sampling. +model : int + If equal to 2, indicates that the model uses skip-grams. +bucket : int + The number of buckets. +min_count : int + The threshold below which the model ignores terms. +t : float + The sample threshold. +minn : int + The minimum ngram length. +maxn : int + The maximum ngram length. +raw_vocab : collections.OrderedDict + A map from words (str) to their frequency (int). The order in the dict + corresponds to the order of the words in the Facebook binary. +nwords : int + The number of words. +vocab_size : int + The size of the vocabulary. +vectors_ngrams : numpy.array + This is a matrix that contains vectors learned by the model. + Each row corresponds to a vector. + The number of vectors is equal to the number of words plus the number of buckets. + The number of columns is equal to the vector dimensionality. +hidden_output : numpy.array + This is a matrix that contains the shallow neural network output. + This array has the same dimensions as vectors_ngrams. + +""" def _struct_unpack(fin, fmt): From 53497a6115a94873956ff65251e61a8d95172d7e Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Sat, 19 Jan 2019 22:31:47 +1100 Subject: [PATCH 02/32] add option to skip hidden matrix loading --- gensim/models/_fasttext_bin.py | 17 +++++++++++------ gensim/models/fasttext.py | 21 +++++++++++++++++---- gensim/test/test_fasttext.py | 15 +++++++++++++++ 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/gensim/models/_fasttext_bin.py b/gensim/models/_fasttext_bin.py index c5ac212c4e..b927914511 100644 --- a/gensim/models/_fasttext_bin.py +++ b/gensim/models/_fasttext_bin.py @@ -120,7 +120,7 @@ def _yield_field_names(): hidden_output : numpy.array This is a matrix that contains the shallow neural network output. This array has the same dimensions as vectors_ngrams. - + May be None - in that case, it is impossible to continue training the model. """ @@ -220,7 +220,7 @@ def _load_matrix(fin, new_format=True): return matrix -def load(fin, encoding='utf-8'): +def load(fin, encoding='utf-8', fast=False): """Load a model from a binary stream. Parameters @@ -229,6 +229,9 @@ def load(fin, encoding='utf-8'): The readable binary stream. encoding : str, optional The encoding to use for decoding text + fast : boolean, optional + If True, skips loading the hidden output matrix. This saves a fair bit + of time, but prevents training continuation. Returns ------- @@ -252,10 +255,12 @@ def load(fin, encoding='utf-8'): vectors_ngrams = _load_matrix(fin, new_format=new_format) - hidden_output = _load_matrix(fin, new_format=new_format) - model.update(vectors_ngrams=vectors_ngrams, hidden_output=hidden_output) - - assert fin.read() == b'', 'expected to reach EOF' + if fast: + hidden_output = None + else: + hidden_output = _load_matrix(fin, new_format=new_format) + assert fin.read() == b'', 'expected to reach EOF' + model.update(vectors_ngrams=vectors_ngrams, hidden_output=hidden_output) model = {k: v for k, v in model.items() if k in _FIELD_NAMES} return Model(**model) diff --git a/gensim/models/fasttext.py b/gensim/models/fasttext.py index 6cdad26fd0..4afdec230d 100644 --- a/gensim/models/fasttext.py +++ b/gensim/models/fasttext.py @@ -704,6 +704,13 @@ def train(self, sentences=None, corpus_file=None, total_examples=None, total_wor >>> model.train(sentences, total_examples=model.corpus_count, epochs=model.epochs) """ + cant_train = hasattr(self.trainables, 'syn1neg') and self.trainables.syn1neg is None + if cant_train: + raise ValueError( + 'this model cannot be trained any further, ' + 'if this is a native model, try loading it with fast=False' + ) + super(FastText, self).train( sentences=sentences, corpus_file=corpus_file, total_examples=total_examples, total_words=total_words, epochs=epochs, start_alpha=start_alpha, end_alpha=end_alpha, word_count=word_count, @@ -754,7 +761,7 @@ def __contains__(self, word): return self.wv.__contains__(word) @classmethod - def load_fasttext_format(cls, model_file, encoding='utf8'): + def load_fasttext_format(cls, model_file, encoding='utf8', fast=False): """Load the input-hidden weight matrix from Facebook's native fasttext `.bin` and `.vec` output files. Notes @@ -770,6 +777,9 @@ def load_fasttext_format(cls, model_file, encoding='utf8'): as Gensim requires only `.bin` file to the load entire fastText model. encoding : str, optional Specifies the file encoding. + fast : boolean, optional + If True, skips loading the hidden output matrix. This saves a fair bit + of time, but prevents training continuation. Returns ------- @@ -777,7 +787,7 @@ def load_fasttext_format(cls, model_file, encoding='utf8'): The loaded model. """ - return _load_fasttext_format(model_file, encoding=encoding) + return _load_fasttext_format(model_file, encoding=encoding, fast=fast) def load_binary_data(self, encoding='utf8'): """Load data from a binary file created by Facebook's native FastText. @@ -959,7 +969,7 @@ def _pad_ones(m, new_shape): return vstack([m, suffix]) -def _load_fasttext_format(model_file, encoding='utf-8'): +def _load_fasttext_format(model_file, encoding='utf-8', fast=False): """Load the input-hidden weight matrix from Facebook's native fasttext `.bin` and `.vec` output files. Parameters @@ -971,6 +981,9 @@ def _load_fasttext_format(model_file, encoding='utf-8'): as Gensim requires only `.bin` file to the load entire fastText model. encoding : str, optional Specifies the file encoding. + fast : boolean, optional + If True, skips loading the hidden output matrix. This saves a fair bit + of time, but prevents training continuation. Returns ------- @@ -980,7 +993,7 @@ def _load_fasttext_format(model_file, encoding='utf-8'): if not model_file.endswith('.bin'): model_file += '.bin' with smart_open(model_file, 'rb') as fin: - m = gensim.models._fasttext_bin.load(fin, encoding=encoding) + m = gensim.models._fasttext_bin.load(fin, encoding=encoding, fast=fast) model = FastText( size=m.dim, diff --git a/gensim/test/test_fasttext.py b/gensim/test/test_fasttext.py index 67b035549b..a1447e51e0 100644 --- a/gensim/test/test_fasttext.py +++ b/gensim/test/test_fasttext.py @@ -59,6 +59,21 @@ def setUp(self): self.test_model = FT_gensim.load_fasttext_format(self.test_model_file) self.test_new_model_file = datapath('lee_fasttext_new') + def test_native_fast(self): + """Can we skip loading the NN and still get a working model?""" + model = FT_gensim.load_fasttext_format(self.test_model_file, fast=True) + + # + # Training continuation should be impossible + # + self.assertIsNone(model.trainables.syn1neg) + self.assertRaises(ValueError, model.train, sentences, + total_examples=model.corpus_count, epochs=model.epochs) + + model.wv['green'] + model.wv['foobar'] + model.wv['thisworddoesnotexist'] + def test_training(self): model = FT_gensim(size=10, min_count=1, hs=1, negative=0, seed=42, workers=1) model.build_vocab(sentences) From 6ae0bb9bdc154e3b04c9b701a87504a910b5591d Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Sun, 20 Jan 2019 01:14:51 +1100 Subject: [PATCH 03/32] review response: rename fast -> full_model --- gensim/models/_fasttext_bin.py | 8 ++++---- gensim/models/fasttext.py | 18 +++++++++--------- gensim/test/test_fasttext.py | 4 ++-- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/gensim/models/_fasttext_bin.py b/gensim/models/_fasttext_bin.py index b927914511..2cbc530a96 100644 --- a/gensim/models/_fasttext_bin.py +++ b/gensim/models/_fasttext_bin.py @@ -220,7 +220,7 @@ def _load_matrix(fin, new_format=True): return matrix -def load(fin, encoding='utf-8', fast=False): +def load(fin, encoding='utf-8', full_model=True): """Load a model from a binary stream. Parameters @@ -229,8 +229,8 @@ def load(fin, encoding='utf-8', fast=False): The readable binary stream. encoding : str, optional The encoding to use for decoding text - fast : boolean, optional - If True, skips loading the hidden output matrix. This saves a fair bit + full_model : boolean, optional + If False, skips loading the hidden output matrix. This saves a fair bit of time, but prevents training continuation. Returns @@ -255,7 +255,7 @@ def load(fin, encoding='utf-8', fast=False): vectors_ngrams = _load_matrix(fin, new_format=new_format) - if fast: + if not full_model: hidden_output = None else: hidden_output = _load_matrix(fin, new_format=new_format) diff --git a/gensim/models/fasttext.py b/gensim/models/fasttext.py index 4afdec230d..f25ff4481d 100644 --- a/gensim/models/fasttext.py +++ b/gensim/models/fasttext.py @@ -708,7 +708,7 @@ def train(self, sentences=None, corpus_file=None, total_examples=None, total_wor if cant_train: raise ValueError( 'this model cannot be trained any further, ' - 'if this is a native model, try loading it with fast=False' + 'if this is a native model, try loading it with full_model=True' ) super(FastText, self).train( @@ -761,7 +761,7 @@ def __contains__(self, word): return self.wv.__contains__(word) @classmethod - def load_fasttext_format(cls, model_file, encoding='utf8', fast=False): + def load_fasttext_format(cls, model_file, encoding='utf8', full_model=True): """Load the input-hidden weight matrix from Facebook's native fasttext `.bin` and `.vec` output files. Notes @@ -777,8 +777,8 @@ def load_fasttext_format(cls, model_file, encoding='utf8', fast=False): as Gensim requires only `.bin` file to the load entire fastText model. encoding : str, optional Specifies the file encoding. - fast : boolean, optional - If True, skips loading the hidden output matrix. This saves a fair bit + full_model : boolean, optional + If False, skips loading the hidden output matrix. This saves a fair bit of time, but prevents training continuation. Returns @@ -787,7 +787,7 @@ def load_fasttext_format(cls, model_file, encoding='utf8', fast=False): The loaded model. """ - return _load_fasttext_format(model_file, encoding=encoding, fast=fast) + return _load_fasttext_format(model_file, encoding=encoding, full_model=full_model) def load_binary_data(self, encoding='utf8'): """Load data from a binary file created by Facebook's native FastText. @@ -969,7 +969,7 @@ def _pad_ones(m, new_shape): return vstack([m, suffix]) -def _load_fasttext_format(model_file, encoding='utf-8', fast=False): +def _load_fasttext_format(model_file, encoding='utf-8', full_model=True): """Load the input-hidden weight matrix from Facebook's native fasttext `.bin` and `.vec` output files. Parameters @@ -981,8 +981,8 @@ def _load_fasttext_format(model_file, encoding='utf-8', fast=False): as Gensim requires only `.bin` file to the load entire fastText model. encoding : str, optional Specifies the file encoding. - fast : boolean, optional - If True, skips loading the hidden output matrix. This saves a fair bit + full_model : boolean, optional + If False, skips loading the hidden output matrix. This saves a fair bit of time, but prevents training continuation. Returns @@ -993,7 +993,7 @@ def _load_fasttext_format(model_file, encoding='utf-8', fast=False): if not model_file.endswith('.bin'): model_file += '.bin' with smart_open(model_file, 'rb') as fin: - m = gensim.models._fasttext_bin.load(fin, encoding=encoding, fast=fast) + m = gensim.models._fasttext_bin.load(fin, encoding=encoding, full_model=full_model) model = FastText( size=m.dim, diff --git a/gensim/test/test_fasttext.py b/gensim/test/test_fasttext.py index a1447e51e0..5437a9b3c8 100644 --- a/gensim/test/test_fasttext.py +++ b/gensim/test/test_fasttext.py @@ -59,9 +59,9 @@ def setUp(self): self.test_model = FT_gensim.load_fasttext_format(self.test_model_file) self.test_new_model_file = datapath('lee_fasttext_new') - def test_native_fast(self): + def test_native_partial_model(self): """Can we skip loading the NN and still get a working model?""" - model = FT_gensim.load_fasttext_format(self.test_model_file, fast=True) + model = FT_gensim.load_fasttext_format(self.test_model_file, full_model=False) # # Training continuation should be impossible From 3406bf091b676343e04830671b512172dd2e6f91 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Sun, 20 Jan 2019 19:38:46 +1100 Subject: [PATCH 04/32] speed up hash function based on ideas from @horpto and @menshikh-iv --- gensim/models/_utils_any2vec.c | 1088 ++++++++++++++++++++---------- gensim/models/_utils_any2vec.pyx | 31 + gensim/models/fasttext.py | 10 +- gensim/models/keyedvectors.py | 38 +- gensim/models/utils_any2vec.py | 126 ++-- gensim/test/test_utils.py | 18 +- 6 files changed, 885 insertions(+), 426 deletions(-) diff --git a/gensim/models/_utils_any2vec.c b/gensim/models/_utils_any2vec.c index c42dd6ef33..d78f295799 100644 --- a/gensim/models/_utils_any2vec.c +++ b/gensim/models/_utils_any2vec.c @@ -608,6 +608,7 @@ static CYTHON_INLINE float __PYX_NAN() { #define __PYX_HAVE__gensim__models___utils_any2vec #define __PYX_HAVE_API__gensim__models___utils_any2vec /* Early includes */ +#include #include #include #include "numpy/arrayobject.h" @@ -847,7 +848,7 @@ static const char *__pyx_f[] = { "type.pxd", }; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":776 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":776 * # in Cython to enable them only on the right systems. * * ctypedef npy_int8 int8_t # <<<<<<<<<<<<<< @@ -856,7 +857,7 @@ static const char *__pyx_f[] = { */ typedef npy_int8 __pyx_t_5numpy_int8_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":777 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":777 * * ctypedef npy_int8 int8_t * ctypedef npy_int16 int16_t # <<<<<<<<<<<<<< @@ -865,7 +866,7 @@ typedef npy_int8 __pyx_t_5numpy_int8_t; */ typedef npy_int16 __pyx_t_5numpy_int16_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":778 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":778 * ctypedef npy_int8 int8_t * ctypedef npy_int16 int16_t * ctypedef npy_int32 int32_t # <<<<<<<<<<<<<< @@ -874,7 +875,7 @@ typedef npy_int16 __pyx_t_5numpy_int16_t; */ typedef npy_int32 __pyx_t_5numpy_int32_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":779 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":779 * ctypedef npy_int16 int16_t * ctypedef npy_int32 int32_t * ctypedef npy_int64 int64_t # <<<<<<<<<<<<<< @@ -883,7 +884,7 @@ typedef npy_int32 __pyx_t_5numpy_int32_t; */ typedef npy_int64 __pyx_t_5numpy_int64_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":783 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":783 * #ctypedef npy_int128 int128_t * * ctypedef npy_uint8 uint8_t # <<<<<<<<<<<<<< @@ -892,7 +893,7 @@ typedef npy_int64 __pyx_t_5numpy_int64_t; */ typedef npy_uint8 __pyx_t_5numpy_uint8_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":784 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":784 * * ctypedef npy_uint8 uint8_t * ctypedef npy_uint16 uint16_t # <<<<<<<<<<<<<< @@ -901,7 +902,7 @@ typedef npy_uint8 __pyx_t_5numpy_uint8_t; */ typedef npy_uint16 __pyx_t_5numpy_uint16_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":785 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":785 * ctypedef npy_uint8 uint8_t * ctypedef npy_uint16 uint16_t * ctypedef npy_uint32 uint32_t # <<<<<<<<<<<<<< @@ -910,7 +911,7 @@ typedef npy_uint16 __pyx_t_5numpy_uint16_t; */ typedef npy_uint32 __pyx_t_5numpy_uint32_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":786 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":786 * ctypedef npy_uint16 uint16_t * ctypedef npy_uint32 uint32_t * ctypedef npy_uint64 uint64_t # <<<<<<<<<<<<<< @@ -919,7 +920,7 @@ typedef npy_uint32 __pyx_t_5numpy_uint32_t; */ typedef npy_uint64 __pyx_t_5numpy_uint64_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":790 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":790 * #ctypedef npy_uint128 uint128_t * * ctypedef npy_float32 float32_t # <<<<<<<<<<<<<< @@ -928,7 +929,7 @@ typedef npy_uint64 __pyx_t_5numpy_uint64_t; */ typedef npy_float32 __pyx_t_5numpy_float32_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":791 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":791 * * ctypedef npy_float32 float32_t * ctypedef npy_float64 float64_t # <<<<<<<<<<<<<< @@ -937,7 +938,7 @@ typedef npy_float32 __pyx_t_5numpy_float32_t; */ typedef npy_float64 __pyx_t_5numpy_float64_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":800 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":800 * # The int types are mapped a bit surprising -- * # numpy.int corresponds to 'l' and numpy.long to 'q' * ctypedef npy_long int_t # <<<<<<<<<<<<<< @@ -946,7 +947,7 @@ typedef npy_float64 __pyx_t_5numpy_float64_t; */ typedef npy_long __pyx_t_5numpy_int_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":801 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":801 * # numpy.int corresponds to 'l' and numpy.long to 'q' * ctypedef npy_long int_t * ctypedef npy_longlong long_t # <<<<<<<<<<<<<< @@ -955,7 +956,7 @@ typedef npy_long __pyx_t_5numpy_int_t; */ typedef npy_longlong __pyx_t_5numpy_long_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":802 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":802 * ctypedef npy_long int_t * ctypedef npy_longlong long_t * ctypedef npy_longlong longlong_t # <<<<<<<<<<<<<< @@ -964,7 +965,7 @@ typedef npy_longlong __pyx_t_5numpy_long_t; */ typedef npy_longlong __pyx_t_5numpy_longlong_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":804 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":804 * ctypedef npy_longlong longlong_t * * ctypedef npy_ulong uint_t # <<<<<<<<<<<<<< @@ -973,7 +974,7 @@ typedef npy_longlong __pyx_t_5numpy_longlong_t; */ typedef npy_ulong __pyx_t_5numpy_uint_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":805 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":805 * * ctypedef npy_ulong uint_t * ctypedef npy_ulonglong ulong_t # <<<<<<<<<<<<<< @@ -982,7 +983,7 @@ typedef npy_ulong __pyx_t_5numpy_uint_t; */ typedef npy_ulonglong __pyx_t_5numpy_ulong_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":806 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":806 * ctypedef npy_ulong uint_t * ctypedef npy_ulonglong ulong_t * ctypedef npy_ulonglong ulonglong_t # <<<<<<<<<<<<<< @@ -991,7 +992,7 @@ typedef npy_ulonglong __pyx_t_5numpy_ulong_t; */ typedef npy_ulonglong __pyx_t_5numpy_ulonglong_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":808 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":808 * ctypedef npy_ulonglong ulonglong_t * * ctypedef npy_intp intp_t # <<<<<<<<<<<<<< @@ -1000,7 +1001,7 @@ typedef npy_ulonglong __pyx_t_5numpy_ulonglong_t; */ typedef npy_intp __pyx_t_5numpy_intp_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":809 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":809 * * ctypedef npy_intp intp_t * ctypedef npy_uintp uintp_t # <<<<<<<<<<<<<< @@ -1009,7 +1010,7 @@ typedef npy_intp __pyx_t_5numpy_intp_t; */ typedef npy_uintp __pyx_t_5numpy_uintp_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":811 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":811 * ctypedef npy_uintp uintp_t * * ctypedef npy_double float_t # <<<<<<<<<<<<<< @@ -1018,7 +1019,7 @@ typedef npy_uintp __pyx_t_5numpy_uintp_t; */ typedef npy_double __pyx_t_5numpy_float_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":812 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":812 * * ctypedef npy_double float_t * ctypedef npy_double double_t # <<<<<<<<<<<<<< @@ -1027,7 +1028,7 @@ typedef npy_double __pyx_t_5numpy_float_t; */ typedef npy_double __pyx_t_5numpy_double_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":813 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":813 * ctypedef npy_double float_t * ctypedef npy_double double_t * ctypedef npy_longdouble longdouble_t # <<<<<<<<<<<<<< @@ -1063,7 +1064,7 @@ static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(do /*--- Type declarations ---*/ struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":815 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":815 * ctypedef npy_longdouble longdouble_t * * ctypedef npy_cfloat cfloat_t # <<<<<<<<<<<<<< @@ -1072,7 +1073,7 @@ struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py; */ typedef npy_cfloat __pyx_t_5numpy_cfloat_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":816 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":816 * * ctypedef npy_cfloat cfloat_t * ctypedef npy_cdouble cdouble_t # <<<<<<<<<<<<<< @@ -1081,7 +1082,7 @@ typedef npy_cfloat __pyx_t_5numpy_cfloat_t; */ typedef npy_cdouble __pyx_t_5numpy_cdouble_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":817 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":817 * ctypedef npy_cfloat cfloat_t * ctypedef npy_cdouble cdouble_t * ctypedef npy_clongdouble clongdouble_t # <<<<<<<<<<<<<< @@ -1090,7 +1091,7 @@ typedef npy_cdouble __pyx_t_5numpy_cdouble_t; */ typedef npy_clongdouble __pyx_t_5numpy_clongdouble_t; -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":819 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":819 * ctypedef npy_clongdouble clongdouble_t * * ctypedef npy_cdouble complex_t # <<<<<<<<<<<<<< @@ -1554,6 +1555,9 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); /* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value); +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint32_t(uint32_t value); + /* RealImag.proto */ #if CYTHON_CCOMPLEX #ifdef __cplusplus @@ -1661,6 +1665,9 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_enum__NPY_TYPES(enum NPY_TYPES v /* CIntFromPy.proto */ static CYTHON_INLINE unsigned int __Pyx_PyInt_As_unsigned_int(PyObject *); +/* CIntFromPy.proto */ +static CYTHON_INLINE char __Pyx_PyInt_As_char(PyObject *); + /* CIntFromPy.proto */ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); @@ -1687,6 +1694,8 @@ static int __Pyx_check_binary_version(void); static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); +/* Module declarations from 'libc.stdint' */ + /* Module declarations from 'cpython.buffer' */ /* Module declarations from 'libc.string' */ @@ -1721,6 +1730,7 @@ static PyTypeObject *__pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____objec static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3(PyObject *); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObject *); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *, int __pyx_skip_dispatch); /*proto*/ +static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject *, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObject *, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObject *, unsigned int, unsigned int, int __pyx_skip_dispatch); /*proto*/ static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*)(PyObject *)); /*proto*/ @@ -1803,8 +1813,9 @@ static PyObject *__pyx_kp_u_unknown_dtype_code_in_numpy_pxd; static PyObject *__pyx_n_s_word; static PyObject *__pyx_n_s_wrap; static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string); /* proto */ -static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string); /* proto */ -static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_bytez); /* proto */ +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4ft_hash_broken(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string); /* proto */ +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info); /* proto */ static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(PyObject *__pyx_self, PyObject *__pyx_v_b); /* proto */ @@ -1823,7 +1834,7 @@ static PyObject *__pyx_tuple__10; static PyObject *__pyx_codeobj__11; /* Late includes */ -/* "gensim/models/_utils_any2vec.pyx":15 +/* "gensim/models/_utils_any2vec.pyx":16 * * * cdef _byte_to_int_py3(b): # <<<<<<<<<<<<<< @@ -1836,7 +1847,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3(PyObj __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("_byte_to_int_py3", 0); - /* "gensim/models/_utils_any2vec.pyx":16 + /* "gensim/models/_utils_any2vec.pyx":17 * * cdef _byte_to_int_py3(b): * return b # <<<<<<<<<<<<<< @@ -1848,7 +1859,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3(PyObj __pyx_r = __pyx_v_b; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":15 + /* "gensim/models/_utils_any2vec.pyx":16 * * * cdef _byte_to_int_py3(b): # <<<<<<<<<<<<<< @@ -1863,7 +1874,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3(PyObj return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":18 +/* "gensim/models/_utils_any2vec.pyx":19 * return b * * cdef _byte_to_int_py2(b): # <<<<<<<<<<<<<< @@ -1878,7 +1889,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObj PyObject *__pyx_t_2 = NULL; __Pyx_RefNannySetupContext("_byte_to_int_py2", 0); - /* "gensim/models/_utils_any2vec.pyx":19 + /* "gensim/models/_utils_any2vec.pyx":20 * * cdef _byte_to_int_py2(b): * return ord(b) # <<<<<<<<<<<<<< @@ -1886,14 +1897,14 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObj * _byte_to_int = _byte_to_int_py2 if PY2 else _byte_to_int_py3 */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_Ord(__pyx_v_b); if (unlikely(__pyx_t_1 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 19, __pyx_L1_error) - __pyx_t_2 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 19, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Ord(__pyx_v_b); if (unlikely(__pyx_t_1 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 20, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 20, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":18 + /* "gensim/models/_utils_any2vec.pyx":19 * return b * * cdef _byte_to_int_py2(b): # <<<<<<<<<<<<<< @@ -1912,7 +1923,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObj return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":24 +/* "gensim/models/_utils_any2vec.pyx":25 * * * cpdef ft_hash(unicode string): # <<<<<<<<<<<<<< @@ -1941,7 +1952,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *__py unsigned int __pyx_t_13; __Pyx_RefNannySetupContext("ft_hash", 0); - /* "gensim/models/_utils_any2vec.pyx":40 + /* "gensim/models/_utils_any2vec.pyx":46 * * """ * cdef unsigned int h = 2166136261 # <<<<<<<<<<<<<< @@ -1950,7 +1961,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *__py */ __pyx_v_h = 0x811C9DC5; - /* "gensim/models/_utils_any2vec.pyx":41 + /* "gensim/models/_utils_any2vec.pyx":47 * """ * cdef unsigned int h = 2166136261 * for c in string.encode("utf-8"): # <<<<<<<<<<<<<< @@ -1959,13 +1970,13 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *__py */ if (unlikely(__pyx_v_string == Py_None)) { PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%.30s'", "encode"); - __PYX_ERR(0, 41, __pyx_L1_error) + __PYX_ERR(0, 47, __pyx_L1_error) } - __pyx_t_1 = PyUnicode_AsUTF8String(__pyx_v_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 41, __pyx_L1_error) + __pyx_t_1 = PyUnicode_AsUTF8String(__pyx_v_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 47, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 41, __pyx_L1_error) + __pyx_t_2 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 47, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = Py_TYPE(__pyx_t_2)->tp_iternext; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 41, __pyx_L1_error) + __pyx_t_3 = Py_TYPE(__pyx_t_2)->tp_iternext; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 47, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; for (;;) { { @@ -1974,7 +1985,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *__py PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(0, 41, __pyx_L1_error) + else __PYX_ERR(0, 47, __pyx_L1_error) } break; } @@ -1983,31 +1994,31 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *__py __Pyx_XDECREF_SET(__pyx_v_c, __pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":42 + /* "gensim/models/_utils_any2vec.pyx":48 * cdef unsigned int h = 2166136261 * for c in string.encode("utf-8"): * h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) # <<<<<<<<<<<<<< * h = np.uint32(h * np.uint32(16777619)) * return h */ - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_np); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 42, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_np); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_uint32); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 42, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_uint32); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 42, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_n_s_np); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 42, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_n_s_np); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_uint32); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 42, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_uint32); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_n_s_np); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 42, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_n_s_np); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_9); - __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_n_s_int8); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 42, __pyx_L1_error) + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_n_s_int8); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - __Pyx_GetModuleGlobalName(__pyx_t_11, __pyx_n_s_byte_to_int); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 42, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_11, __pyx_n_s_byte_to_int); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_11); __pyx_t_12 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_11))) { @@ -2021,7 +2032,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *__py } __pyx_t_9 = (__pyx_t_12) ? __Pyx_PyObject_Call2Args(__pyx_t_11, __pyx_t_12, __pyx_v_c) : __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_v_c); __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0; - if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 42, __pyx_L1_error) + if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_9); __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __pyx_t_11 = NULL; @@ -2037,7 +2048,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *__py __pyx_t_7 = (__pyx_t_11) ? __Pyx_PyObject_Call2Args(__pyx_t_10, __pyx_t_11, __pyx_t_9) : __Pyx_PyObject_CallOneArg(__pyx_t_10, __pyx_t_9); __Pyx_XDECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 42, __pyx_L1_error) + if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; __pyx_t_10 = NULL; @@ -2053,10 +2064,10 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *__py __pyx_t_6 = (__pyx_t_10) ? __Pyx_PyObject_Call2Args(__pyx_t_8, __pyx_t_10, __pyx_t_7) : __Pyx_PyObject_CallOneArg(__pyx_t_8, __pyx_t_7); __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 42, __pyx_L1_error) + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_8 = PyNumber_Xor(__pyx_t_4, __pyx_t_6); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 42, __pyx_L1_error) + __pyx_t_8 = PyNumber_Xor(__pyx_t_4, __pyx_t_6); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; @@ -2073,30 +2084,30 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *__py __pyx_t_1 = (__pyx_t_6) ? __Pyx_PyObject_Call2Args(__pyx_t_5, __pyx_t_6, __pyx_t_8) : __Pyx_PyObject_CallOneArg(__pyx_t_5, __pyx_t_8); __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 42, __pyx_L1_error) + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_13 = __Pyx_PyInt_As_unsigned_int(__pyx_t_1); if (unlikely((__pyx_t_13 == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 42, __pyx_L1_error) + __pyx_t_13 = __Pyx_PyInt_As_unsigned_int(__pyx_t_1); if (unlikely((__pyx_t_13 == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 48, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_h = __pyx_t_13; - /* "gensim/models/_utils_any2vec.pyx":43 + /* "gensim/models/_utils_any2vec.pyx":49 * for c in string.encode("utf-8"): * h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) * h = np.uint32(h * np.uint32(16777619)) # <<<<<<<<<<<<<< * return h * */ - __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_n_s_np); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 43, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_n_s_np); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 49, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_uint32); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 43, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_uint32); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 49, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 43, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 49, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_np); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 43, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_np); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 49, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_uint32); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 43, __pyx_L1_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_uint32); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 49, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_4 = NULL; @@ -2111,10 +2122,10 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *__py } __pyx_t_6 = (__pyx_t_4) ? __Pyx_PyObject_Call2Args(__pyx_t_7, __pyx_t_4, __pyx_int_16777619) : __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_int_16777619); __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 43, __pyx_L1_error) + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 49, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = PyNumber_Multiply(__pyx_t_5, __pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 43, __pyx_L1_error) + __pyx_t_7 = PyNumber_Multiply(__pyx_t_5, __pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 49, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; @@ -2131,14 +2142,14 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *__py __pyx_t_1 = (__pyx_t_6) ? __Pyx_PyObject_Call2Args(__pyx_t_8, __pyx_t_6, __pyx_t_7) : __Pyx_PyObject_CallOneArg(__pyx_t_8, __pyx_t_7); __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 43, __pyx_L1_error) + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 49, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_13 = __Pyx_PyInt_As_unsigned_int(__pyx_t_1); if (unlikely((__pyx_t_13 == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 43, __pyx_L1_error) + __pyx_t_13 = __Pyx_PyInt_As_unsigned_int(__pyx_t_1); if (unlikely((__pyx_t_13 == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 49, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_h = __pyx_t_13; - /* "gensim/models/_utils_any2vec.pyx":41 + /* "gensim/models/_utils_any2vec.pyx":47 * """ * cdef unsigned int h = 2166136261 * for c in string.encode("utf-8"): # <<<<<<<<<<<<<< @@ -2148,7 +2159,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *__py } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "gensim/models/_utils_any2vec.pyx":44 + /* "gensim/models/_utils_any2vec.pyx":50 * h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) * h = np.uint32(h * np.uint32(16777619)) * return h # <<<<<<<<<<<<<< @@ -2156,13 +2167,13 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *__py * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 44, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 50, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":24 + /* "gensim/models/_utils_any2vec.pyx":25 * * * cpdef ft_hash(unicode string): # <<<<<<<<<<<<<< @@ -2194,12 +2205,12 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *__py /* Python wrapper */ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash(PyObject *__pyx_self, PyObject *__pyx_v_string); /*proto*/ -static char __pyx_doc_6gensim_6models_14_utils_any2vec_ft_hash[] = "ft_hash(unicode string)\nCalculate hash based on `string`.\n Reproduce `hash method from Facebook fastText implementation\n `_.\n\n Parameters\n ----------\n string : unicode\n The string whose hash needs to be calculated.\n\n Returns\n -------\n unsigned int\n The hash of the string.\n\n "; +static char __pyx_doc_6gensim_6models_14_utils_any2vec_ft_hash[] = "ft_hash(unicode string)\nCalculate hash based on `string`.\n Reproduce `hash method from Facebook fastText implementation\n `_.\n\n This function is expensive because it encodes each input string as UTF-8,\n and this can accumulate to a high computational cost if there are many\n strings. If possible, convert your strings to bytes and use the faster\n :py:func:`ft_hash_bytes` function instead.\n\n Parameters\n ----------\n string : unicode\n The string whose hash needs to be calculated.\n\n Returns\n -------\n unsigned int\n The hash of the string.\n\n "; static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash(PyObject *__pyx_self, PyObject *__pyx_v_string) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("ft_hash (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_string), (&PyUnicode_Type), 1, "string", 1))) __PYX_ERR(0, 24, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_string), (&PyUnicode_Type), 1, "string", 1))) __PYX_ERR(0, 25, __pyx_L1_error) __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash(__pyx_self, ((PyObject*)__pyx_v_string)); /* function exit code */ @@ -2217,7 +2228,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash(CYTHON_UNUSED PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("ft_hash", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(__pyx_v_string, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 24, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(__pyx_v_string, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 25, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -2234,7 +2245,153 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash(CYTHON_UNUSED return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":47 +/* "gensim/models/_utils_any2vec.pyx":53 + * + * + * cpdef ft_hash_bytes(bytes bytez): # <<<<<<<<<<<<<< + * """Calculate hash based on `bytez`. + * Reproduce `hash method from Facebook fastText implementation + */ + +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_bytes(PyObject *__pyx_self, PyObject *__pyx_v_bytez); /*proto*/ +static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject *__pyx_v_bytez, CYTHON_UNUSED int __pyx_skip_dispatch) { + uint32_t __pyx_v_h; + char __pyx_v_b; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + char *__pyx_t_2; + char *__pyx_t_3; + char *__pyx_t_4; + char *__pyx_t_5; + PyObject *__pyx_t_6 = NULL; + __Pyx_RefNannySetupContext("ft_hash_bytes", 0); + + /* "gensim/models/_utils_any2vec.pyx":69 + * + * """ + * cdef uint32_t h = 2166136261 # <<<<<<<<<<<<<< + * cdef char b + * + */ + __pyx_v_h = 0x811C9DC5; + + /* "gensim/models/_utils_any2vec.pyx":72 + * cdef char b + * + * for b in bytez: # <<<<<<<<<<<<<< + * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss + * h = h * 16777619 + */ + if (unlikely(__pyx_v_bytez == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); + __PYX_ERR(0, 72, __pyx_L1_error) + } + __Pyx_INCREF(__pyx_v_bytez); + __pyx_t_1 = __pyx_v_bytez; + __pyx_t_3 = PyBytes_AS_STRING(__pyx_t_1); + __pyx_t_4 = (__pyx_t_3 + PyBytes_GET_SIZE(__pyx_t_1)); + for (__pyx_t_5 = __pyx_t_3; __pyx_t_5 < __pyx_t_4; __pyx_t_5++) { + __pyx_t_2 = __pyx_t_5; + __pyx_v_b = (__pyx_t_2[0]); + + /* "gensim/models/_utils_any2vec.pyx":73 + * + * for b in bytez: + * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss # <<<<<<<<<<<<<< + * h = h * 16777619 + * return h + */ + __pyx_v_h = (__pyx_v_h ^ ((uint32_t)((int8_t)__pyx_v_b))); + + /* "gensim/models/_utils_any2vec.pyx":74 + * for b in bytez: + * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss + * h = h * 16777619 # <<<<<<<<<<<<<< + * return h + * + */ + __pyx_v_h = (__pyx_v_h * 0x1000193); + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "gensim/models/_utils_any2vec.pyx":75 + * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss + * h = h * 16777619 + * return h # <<<<<<<<<<<<<< + * + * + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_6 = __Pyx_PyInt_From_uint32_t(__pyx_v_h); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 75, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; + goto __pyx_L0; + + /* "gensim/models/_utils_any2vec.pyx":53 + * + * + * cpdef ft_hash_bytes(bytes bytez): # <<<<<<<<<<<<<< + * """Calculate hash based on `bytez`. + * Reproduce `hash method from Facebook fastText implementation + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("gensim.models._utils_any2vec.ft_hash_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_bytes(PyObject *__pyx_self, PyObject *__pyx_v_bytez); /*proto*/ +static char __pyx_doc_6gensim_6models_14_utils_any2vec_2ft_hash_bytes[] = "ft_hash_bytes(bytes bytez)\nCalculate hash based on `bytez`.\n Reproduce `hash method from Facebook fastText implementation\n `_.\n\n Parameters\n ----------\n bytez : bytes\n The string whose hash needs to be calculated, encoded as UTF-8.\n\n Returns\n -------\n unsigned int\n The hash of the string.\n\n "; +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_bytes(PyObject *__pyx_self, PyObject *__pyx_v_bytez) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("ft_hash_bytes (wrapper)", 0); + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_bytez), (&PyBytes_Type), 1, "bytez", 1))) __PYX_ERR(0, 53, __pyx_L1_error) + __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_bytes(__pyx_self, ((PyObject*)__pyx_v_bytez)); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_bytez) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("ft_hash_bytes", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(__pyx_v_bytez, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 53, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("gensim.models._utils_any2vec.ft_hash_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "gensim/models/_utils_any2vec.pyx":78 * * * cpdef ft_hash_broken(unicode string): # <<<<<<<<<<<<<< @@ -2242,7 +2399,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash(CYTHON_UNUSED * */ -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken(PyObject *__pyx_self, PyObject *__pyx_v_string); /*proto*/ +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5ft_hash_broken(PyObject *__pyx_self, PyObject *__pyx_v_string); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObject *__pyx_v_string, CYTHON_UNUSED int __pyx_skip_dispatch) { unsigned int __pyx_v_h; PyObject *__pyx_v_c = NULL; @@ -2259,7 +2416,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec long __pyx_t_9; __Pyx_RefNannySetupContext("ft_hash_broken", 0); - /* "gensim/models/_utils_any2vec.pyx":64 + /* "gensim/models/_utils_any2vec.pyx":95 * * """ * cdef unsigned int h = 2166136261 # <<<<<<<<<<<<<< @@ -2268,7 +2425,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec */ __pyx_v_h = 0x811C9DC5; - /* "gensim/models/_utils_any2vec.pyx":65 + /* "gensim/models/_utils_any2vec.pyx":96 * """ * cdef unsigned int h = 2166136261 * for c in string: # <<<<<<<<<<<<<< @@ -2277,29 +2434,29 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec */ if (unlikely(__pyx_v_string == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); - __PYX_ERR(0, 65, __pyx_L1_error) + __PYX_ERR(0, 96, __pyx_L1_error) } __Pyx_INCREF(__pyx_v_string); __pyx_t_1 = __pyx_v_string; - __pyx_t_6 = __Pyx_init_unicode_iteration(__pyx_t_1, (&__pyx_t_3), (&__pyx_t_4), (&__pyx_t_5)); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 65, __pyx_L1_error) + __pyx_t_6 = __Pyx_init_unicode_iteration(__pyx_t_1, (&__pyx_t_3), (&__pyx_t_4), (&__pyx_t_5)); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 96, __pyx_L1_error) for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_3; __pyx_t_7++) { __pyx_t_2 = __pyx_t_7; - __pyx_t_8 = PyUnicode_FromOrdinal(__Pyx_PyUnicode_READ(__pyx_t_5, __pyx_t_4, __pyx_t_2)); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 65, __pyx_L1_error) + __pyx_t_8 = PyUnicode_FromOrdinal(__Pyx_PyUnicode_READ(__pyx_t_5, __pyx_t_4, __pyx_t_2)); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 96, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_XDECREF_SET(__pyx_v_c, __pyx_t_8); __pyx_t_8 = 0; - /* "gensim/models/_utils_any2vec.pyx":66 + /* "gensim/models/_utils_any2vec.pyx":97 * cdef unsigned int h = 2166136261 * for c in string: * h ^= ord(c) # <<<<<<<<<<<<<< * h *= 16777619 * return h */ - __pyx_t_9 = __Pyx_PyObject_Ord(__pyx_v_c); if (unlikely(__pyx_t_9 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 66, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyObject_Ord(__pyx_v_c); if (unlikely(__pyx_t_9 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 97, __pyx_L1_error) __pyx_v_h = (__pyx_v_h ^ __pyx_t_9); - /* "gensim/models/_utils_any2vec.pyx":67 + /* "gensim/models/_utils_any2vec.pyx":98 * for c in string: * h ^= ord(c) * h *= 16777619 # <<<<<<<<<<<<<< @@ -2310,7 +2467,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":68 + /* "gensim/models/_utils_any2vec.pyx":99 * h ^= ord(c) * h *= 16777619 * return h # <<<<<<<<<<<<<< @@ -2318,13 +2475,13 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_8 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 68, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 99, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __pyx_r = __pyx_t_8; __pyx_t_8 = 0; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":47 + /* "gensim/models/_utils_any2vec.pyx":78 * * * cpdef ft_hash_broken(unicode string): # <<<<<<<<<<<<<< @@ -2346,14 +2503,14 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec } /* Python wrapper */ -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken(PyObject *__pyx_self, PyObject *__pyx_v_string); /*proto*/ -static char __pyx_doc_6gensim_6models_14_utils_any2vec_2ft_hash_broken[] = "ft_hash_broken(unicode string)\nCalculate hash based on `string`.\n\n This implementation is broken, see https://github.com/RaRe-Technologies/gensim/issues/2059.\n It is here only for maintaining backwards compatibility with older models.\n\n Parameters\n ----------\n string : unicode\n The string whose hash needs to be calculated.\n\n Returns\n -------\n unsigned int\n The hash of the string.\n\n "; -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken(PyObject *__pyx_self, PyObject *__pyx_v_string) { +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5ft_hash_broken(PyObject *__pyx_self, PyObject *__pyx_v_string); /*proto*/ +static char __pyx_doc_6gensim_6models_14_utils_any2vec_4ft_hash_broken[] = "ft_hash_broken(unicode string)\nCalculate hash based on `string`.\n\n This implementation is broken, see https://github.com/RaRe-Technologies/gensim/issues/2059.\n It is here only for maintaining backwards compatibility with older models.\n\n Parameters\n ----------\n string : unicode\n The string whose hash needs to be calculated.\n\n Returns\n -------\n unsigned int\n The hash of the string.\n\n "; +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5ft_hash_broken(PyObject *__pyx_self, PyObject *__pyx_v_string) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("ft_hash_broken (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_string), (&PyUnicode_Type), 1, "string", 1))) __PYX_ERR(0, 47, __pyx_L1_error) - __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(__pyx_self, ((PyObject*)__pyx_v_string)); + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_string), (&PyUnicode_Type), 1, "string", 1))) __PYX_ERR(0, 78, __pyx_L1_error) + __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_4ft_hash_broken(__pyx_self, ((PyObject*)__pyx_v_string)); /* function exit code */ goto __pyx_L0; @@ -2364,13 +2521,13 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken(PyObj return __pyx_r; } -static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string) { +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4ft_hash_broken(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("ft_hash_broken", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(__pyx_v_string, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 47, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(__pyx_v_string, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 78, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -2387,7 +2544,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHO return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":71 +/* "gensim/models/_utils_any2vec.pyx":102 * * * cpdef compute_ngrams(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -2395,7 +2552,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHO * */ -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n, CYTHON_UNUSED int __pyx_skip_dispatch) { PyObject *__pyx_v_extended_word = 0; PyObject *__pyx_v_ngrams = NULL; @@ -2419,14 +2576,14 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec int __pyx_t_14; __Pyx_RefNannySetupContext("compute_ngrams", 0); - /* "gensim/models/_utils_any2vec.pyx":89 + /* "gensim/models/_utils_any2vec.pyx":120 * * """ * cdef unicode extended_word = f'<{word}>' # <<<<<<<<<<<<<< * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): */ - __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 89, __pyx_L1_error) + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 120, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = 0; __pyx_t_3 = 127; @@ -2434,7 +2591,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __pyx_t_2 += 1; __Pyx_GIVEREF(__pyx_kp_u_); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_kp_u_); - __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_word, __pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 89, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_word, __pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 120, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = (__Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) > __pyx_t_3) ? __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) : __pyx_t_3; __pyx_t_2 += __Pyx_PyUnicode_GET_LENGTH(__pyx_t_4); @@ -2445,43 +2602,43 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __pyx_t_2 += 1; __Pyx_GIVEREF(__pyx_kp_u__2); PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_kp_u__2); - __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_1, 3, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 89, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_1, 3, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 120, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_extended_word = ((PyObject*)__pyx_t_4); __pyx_t_4 = 0; - /* "gensim/models/_utils_any2vec.pyx":90 + /* "gensim/models/_utils_any2vec.pyx":121 * """ * cdef unicode extended_word = f'<{word}>' * ngrams = [] # <<<<<<<<<<<<<< * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): */ - __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 90, __pyx_L1_error) + __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 121, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_v_ngrams = ((PyObject*)__pyx_t_4); __pyx_t_4 = 0; - /* "gensim/models/_utils_any2vec.pyx":91 + /* "gensim/models/_utils_any2vec.pyx":122 * cdef unicode extended_word = f'<{word}>' * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): # <<<<<<<<<<<<<< * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) */ - __pyx_t_4 = __Pyx_PyInt_From_unsigned_int(__pyx_v_min_n); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 91, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyInt_From_unsigned_int(__pyx_v_min_n); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 122, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = __pyx_v_max_n; - __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 91, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 122, __pyx_L1_error) if (((__pyx_t_5 < __pyx_t_2) != 0)) { __pyx_t_6 = __pyx_t_5; } else { __pyx_t_6 = __pyx_t_2; } - __pyx_t_1 = PyInt_FromSsize_t((__pyx_t_6 + 1)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 91, __pyx_L1_error) + __pyx_t_1 = PyInt_FromSsize_t((__pyx_t_6 + 1)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 122, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 91, __pyx_L1_error) + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 122, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_4); @@ -2489,16 +2646,16 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_1); __pyx_t_4 = 0; __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 91, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 122, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (likely(PyList_CheckExact(__pyx_t_1)) || PyTuple_CheckExact(__pyx_t_1)) { __pyx_t_7 = __pyx_t_1; __Pyx_INCREF(__pyx_t_7); __pyx_t_6 = 0; __pyx_t_8 = NULL; } else { - __pyx_t_6 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 91, __pyx_L1_error) + __pyx_t_6 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 122, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_8 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 91, __pyx_L1_error) + __pyx_t_8 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 122, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; for (;;) { @@ -2506,17 +2663,17 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec if (likely(PyList_CheckExact(__pyx_t_7))) { if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_7)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 91, __pyx_L1_error) + __pyx_t_1 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 122, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 91, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 122, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } else { if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_7)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 91, __pyx_L1_error) + __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 122, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 91, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 122, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } @@ -2526,7 +2683,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(0, 91, __pyx_L1_error) + else __PYX_ERR(0, 122, __pyx_L1_error) } break; } @@ -2535,23 +2692,23 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __Pyx_XDECREF_SET(__pyx_v_ngram_length, __pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":92 + /* "gensim/models/_utils_any2vec.pyx":123 * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): # <<<<<<<<<<<<<< * ngrams.append(extended_word[i:i + ngram_length]) * return ngrams */ - __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 92, __pyx_L1_error) - __pyx_t_1 = PyInt_FromSsize_t(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 123, __pyx_L1_error) + __pyx_t_1 = PyInt_FromSsize_t(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 123, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_4 = PyNumber_Subtract(__pyx_t_1, __pyx_v_ngram_length); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_t_4 = PyNumber_Subtract(__pyx_t_1, __pyx_v_ngram_length); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 123, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyInt_AddObjC(__pyx_t_4, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_AddObjC(__pyx_t_4, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 123, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 123, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_INCREF(__pyx_int_0); __Pyx_GIVEREF(__pyx_int_0); @@ -2559,16 +2716,16 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 123, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (likely(PyList_CheckExact(__pyx_t_1)) || PyTuple_CheckExact(__pyx_t_1)) { __pyx_t_4 = __pyx_t_1; __Pyx_INCREF(__pyx_t_4); __pyx_t_2 = 0; __pyx_t_9 = NULL; } else { - __pyx_t_2 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_t_2 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 123, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_9 = Py_TYPE(__pyx_t_4)->tp_iternext; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_t_9 = Py_TYPE(__pyx_t_4)->tp_iternext; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 123, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; for (;;) { @@ -2576,17 +2733,17 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec if (likely(PyList_CheckExact(__pyx_t_4))) { if (__pyx_t_2 >= PyList_GET_SIZE(__pyx_t_4)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_t_1 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 123, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 123, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } else { if (__pyx_t_2 >= PyTuple_GET_SIZE(__pyx_t_4)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 123, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 123, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } @@ -2596,7 +2753,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(0, 92, __pyx_L1_error) + else __PYX_ERR(0, 123, __pyx_L1_error) } break; } @@ -2605,7 +2762,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __Pyx_XDECREF_SET(__pyx_v_i, __pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":93 + /* "gensim/models/_utils_any2vec.pyx":124 * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) # <<<<<<<<<<<<<< @@ -2617,26 +2774,26 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec if (__pyx_t_11) { __pyx_t_10 = 0; } else { - __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 93, __pyx_L1_error) + __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 124, __pyx_L1_error) __pyx_t_10 = __pyx_t_12; } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyNumber_Add(__pyx_v_i, __pyx_v_ngram_length); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 93, __pyx_L1_error) + __pyx_t_1 = PyNumber_Add(__pyx_v_i, __pyx_v_ngram_length); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 124, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_11 = (__pyx_t_1 == Py_None); if (__pyx_t_11) { __pyx_t_12 = PY_SSIZE_T_MAX; } else { - __pyx_t_13 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_13 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 93, __pyx_L1_error) + __pyx_t_13 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_13 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 124, __pyx_L1_error) __pyx_t_12 = __pyx_t_13; } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyUnicode_Substring(__pyx_v_extended_word, __pyx_t_10, __pyx_t_12); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 93, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyUnicode_Substring(__pyx_v_extended_word, __pyx_t_10, __pyx_t_12); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 124, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_1); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 93, __pyx_L1_error) + __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_1); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 124, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":92 + /* "gensim/models/_utils_any2vec.pyx":123 * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): # <<<<<<<<<<<<<< @@ -2646,7 +2803,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec } __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "gensim/models/_utils_any2vec.pyx":91 + /* "gensim/models/_utils_any2vec.pyx":122 * cdef unicode extended_word = f'<{word}>' * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): # <<<<<<<<<<<<<< @@ -2656,7 +2813,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - /* "gensim/models/_utils_any2vec.pyx":94 + /* "gensim/models/_utils_any2vec.pyx":125 * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) * return ngrams # <<<<<<<<<<<<<< @@ -2666,7 +2823,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __pyx_r = __pyx_v_ngrams; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":71 + /* "gensim/models/_utils_any2vec.pyx":102 * * * cpdef compute_ngrams(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -2692,9 +2849,9 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec } /* Python wrapper */ -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_6gensim_6models_14_utils_any2vec_4compute_ngrams[] = "compute_ngrams(word, unsigned int min_n, unsigned int max_n)\nGet the list of all possible ngrams for a given word.\n\n Parameters\n ----------\n word : str\n The word whose ngrams need to be computed.\n min_n : unsigned int\n Minimum character length of the ngrams.\n max_n : unsigned int\n Maximum character length of the ngrams.\n\n Returns\n -------\n list of str\n Sequence of character ngrams.\n\n "; -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_6gensim_6models_14_utils_any2vec_6compute_ngrams[] = "compute_ngrams(word, unsigned int min_n, unsigned int max_n)\nGet the list of all possible ngrams for a given word.\n\n Parameters\n ----------\n word : str\n The word whose ngrams need to be computed.\n min_n : unsigned int\n Minimum character length of the ngrams.\n max_n : unsigned int\n Maximum character length of the ngrams.\n\n Returns\n -------\n list of str\n Sequence of character ngrams.\n\n "; +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_word = 0; unsigned int __pyx_v_min_n; unsigned int __pyx_v_max_n; @@ -2726,17 +2883,17 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams(PyObj case 1: if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_min_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 1); __PYX_ERR(0, 71, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 1); __PYX_ERR(0, 102, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 2); __PYX_ERR(0, 71, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 2); __PYX_ERR(0, 102, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams") < 0)) __PYX_ERR(0, 71, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams") < 0)) __PYX_ERR(0, 102, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { goto __pyx_L5_argtuple_error; @@ -2746,31 +2903,31 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams(PyObj values[2] = PyTuple_GET_ITEM(__pyx_args, 2); } __pyx_v_word = values[0]; - __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 71, __pyx_L3_error) - __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 71, __pyx_L3_error) + __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 102, __pyx_L3_error) + __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 102, __pyx_L3_error) } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 71, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 102, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(__pyx_self, __pyx_v_word, __pyx_v_min_n, __pyx_v_max_n); + __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams(__pyx_self, __pyx_v_word, __pyx_v_min_n, __pyx_v_max_n); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n) { +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("compute_ngrams", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 71, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 102, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -2787,7 +2944,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHO return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 * # experimental exception made for __getbuffer__ and __releasebuffer__ * # -- the details of this may change. * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< @@ -2836,7 +2993,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); __Pyx_GIVEREF(__pyx_v_info->obj); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":265 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":265 * * cdef int i, ndim * cdef int endian_detector = 1 # <<<<<<<<<<<<<< @@ -2845,7 +3002,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_endian_detector = 1; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":266 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":266 * cdef int i, ndim * cdef int endian_detector = 1 * cdef bint little_endian = ((&endian_detector)[0] != 0) # <<<<<<<<<<<<<< @@ -2854,7 +3011,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_little_endian = ((((char *)(&__pyx_v_endian_detector))[0]) != 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":268 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":268 * cdef bint little_endian = ((&endian_detector)[0] != 0) * * ndim = PyArray_NDIM(self) # <<<<<<<<<<<<<< @@ -2863,7 +3020,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_ndim = PyArray_NDIM(__pyx_v_self); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 * ndim = PyArray_NDIM(self) * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -2877,7 +3034,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P goto __pyx_L4_bool_binop_done; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":271 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":271 * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): # <<<<<<<<<<<<<< @@ -2888,7 +3045,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = __pyx_t_2; __pyx_L4_bool_binop_done:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 * ndim = PyArray_NDIM(self) * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -2897,7 +3054,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ if (unlikely(__pyx_t_1)) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): * raise ValueError(u"ndarray is not C contiguous") # <<<<<<<<<<<<<< @@ -2910,7 +3067,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 272, __pyx_L1_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 * ndim = PyArray_NDIM(self) * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -2919,7 +3076,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 * raise ValueError(u"ndarray is not C contiguous") * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -2933,7 +3090,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P goto __pyx_L7_bool_binop_done; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":275 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":275 * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): # <<<<<<<<<<<<<< @@ -2944,7 +3101,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = __pyx_t_2; __pyx_L7_bool_binop_done:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 * raise ValueError(u"ndarray is not C contiguous") * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -2953,7 +3110,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ if (unlikely(__pyx_t_1)) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): * raise ValueError(u"ndarray is not Fortran contiguous") # <<<<<<<<<<<<<< @@ -2966,7 +3123,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 276, __pyx_L1_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 * raise ValueError(u"ndarray is not C contiguous") * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -2975,7 +3132,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":278 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":278 * raise ValueError(u"ndarray is not Fortran contiguous") * * info.buf = PyArray_DATA(self) # <<<<<<<<<<<<<< @@ -2984,7 +3141,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->buf = PyArray_DATA(__pyx_v_self); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":279 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":279 * * info.buf = PyArray_DATA(self) * info.ndim = ndim # <<<<<<<<<<<<<< @@ -2993,7 +3150,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->ndim = __pyx_v_ndim; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 * info.buf = PyArray_DATA(self) * info.ndim = ndim * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< @@ -3003,7 +3160,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = (((sizeof(npy_intp)) != (sizeof(Py_ssize_t))) != 0); if (__pyx_t_1) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":283 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":283 * # Allocate new buffer for strides and shape info. * # This is allocated as one block, strides first. * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) # <<<<<<<<<<<<<< @@ -3012,7 +3169,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->strides = ((Py_ssize_t *)PyObject_Malloc((((sizeof(Py_ssize_t)) * 2) * ((size_t)__pyx_v_ndim)))); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":284 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":284 * # This is allocated as one block, strides first. * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) * info.shape = info.strides + ndim # <<<<<<<<<<<<<< @@ -3021,7 +3178,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->shape = (__pyx_v_info->strides + __pyx_v_ndim); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":285 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":285 * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) * info.shape = info.strides + ndim * for i in range(ndim): # <<<<<<<<<<<<<< @@ -3033,7 +3190,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { __pyx_v_i = __pyx_t_6; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":286 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":286 * info.shape = info.strides + ndim * for i in range(ndim): * info.strides[i] = PyArray_STRIDES(self)[i] # <<<<<<<<<<<<<< @@ -3042,7 +3199,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ (__pyx_v_info->strides[__pyx_v_i]) = (PyArray_STRIDES(__pyx_v_self)[__pyx_v_i]); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":287 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":287 * for i in range(ndim): * info.strides[i] = PyArray_STRIDES(self)[i] * info.shape[i] = PyArray_DIMS(self)[i] # <<<<<<<<<<<<<< @@ -3052,7 +3209,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P (__pyx_v_info->shape[__pyx_v_i]) = (PyArray_DIMS(__pyx_v_self)[__pyx_v_i]); } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 * info.buf = PyArray_DATA(self) * info.ndim = ndim * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< @@ -3062,7 +3219,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P goto __pyx_L9; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":289 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":289 * info.shape[i] = PyArray_DIMS(self)[i] * else: * info.strides = PyArray_STRIDES(self) # <<<<<<<<<<<<<< @@ -3072,7 +3229,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P /*else*/ { __pyx_v_info->strides = ((Py_ssize_t *)PyArray_STRIDES(__pyx_v_self)); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":290 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":290 * else: * info.strides = PyArray_STRIDES(self) * info.shape = PyArray_DIMS(self) # <<<<<<<<<<<<<< @@ -3083,7 +3240,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P } __pyx_L9:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":291 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":291 * info.strides = PyArray_STRIDES(self) * info.shape = PyArray_DIMS(self) * info.suboffsets = NULL # <<<<<<<<<<<<<< @@ -3092,7 +3249,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->suboffsets = NULL; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":292 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":292 * info.shape = PyArray_DIMS(self) * info.suboffsets = NULL * info.itemsize = PyArray_ITEMSIZE(self) # <<<<<<<<<<<<<< @@ -3101,7 +3258,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->itemsize = PyArray_ITEMSIZE(__pyx_v_self); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":293 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":293 * info.suboffsets = NULL * info.itemsize = PyArray_ITEMSIZE(self) * info.readonly = not PyArray_ISWRITEABLE(self) # <<<<<<<<<<<<<< @@ -3110,7 +3267,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->readonly = (!(PyArray_ISWRITEABLE(__pyx_v_self) != 0)); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":296 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":296 * * cdef int t * cdef char* f = NULL # <<<<<<<<<<<<<< @@ -3119,7 +3276,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_f = NULL; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":297 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":297 * cdef int t * cdef char* f = NULL * cdef dtype descr = PyArray_DESCR(self) # <<<<<<<<<<<<<< @@ -3132,7 +3289,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_v_descr = ((PyArray_Descr *)__pyx_t_3); __pyx_t_3 = 0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":300 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":300 * cdef int offset * * info.obj = self # <<<<<<<<<<<<<< @@ -3145,7 +3302,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = ((PyObject *)__pyx_v_self); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 * info.obj = self * * if not PyDataType_HASFIELDS(descr): # <<<<<<<<<<<<<< @@ -3155,7 +3312,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = ((!(PyDataType_HASFIELDS(__pyx_v_descr) != 0)) != 0); if (__pyx_t_1) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":303 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":303 * * if not PyDataType_HASFIELDS(descr): * t = descr.type_num # <<<<<<<<<<<<<< @@ -3165,7 +3322,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_4 = __pyx_v_descr->type_num; __pyx_v_t = __pyx_t_4; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 * if not PyDataType_HASFIELDS(descr): * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -3185,7 +3342,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P } __pyx_L15_next_or:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":305 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":305 * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or * (descr.byteorder == c'<' and not little_endian)): # <<<<<<<<<<<<<< @@ -3202,7 +3359,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = __pyx_t_2; __pyx_L14_bool_binop_done:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 * if not PyDataType_HASFIELDS(descr): * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -3211,7 +3368,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ if (unlikely(__pyx_t_1)) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 * if ((descr.byteorder == c'>' and little_endian) or * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< @@ -3224,7 +3381,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 306, __pyx_L1_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 * if not PyDataType_HASFIELDS(descr): * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -3233,7 +3390,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":307 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":307 * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") * if t == NPY_BYTE: f = "b" # <<<<<<<<<<<<<< @@ -3246,7 +3403,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_UBYTE: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":308 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":308 * raise ValueError(u"Non-native byte order not supported") * if t == NPY_BYTE: f = "b" * elif t == NPY_UBYTE: f = "B" # <<<<<<<<<<<<<< @@ -3257,7 +3414,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_SHORT: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":309 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":309 * if t == NPY_BYTE: f = "b" * elif t == NPY_UBYTE: f = "B" * elif t == NPY_SHORT: f = "h" # <<<<<<<<<<<<<< @@ -3268,7 +3425,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_USHORT: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":310 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":310 * elif t == NPY_UBYTE: f = "B" * elif t == NPY_SHORT: f = "h" * elif t == NPY_USHORT: f = "H" # <<<<<<<<<<<<<< @@ -3279,7 +3436,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_INT: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":311 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":311 * elif t == NPY_SHORT: f = "h" * elif t == NPY_USHORT: f = "H" * elif t == NPY_INT: f = "i" # <<<<<<<<<<<<<< @@ -3290,7 +3447,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_UINT: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":312 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":312 * elif t == NPY_USHORT: f = "H" * elif t == NPY_INT: f = "i" * elif t == NPY_UINT: f = "I" # <<<<<<<<<<<<<< @@ -3301,7 +3458,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_LONG: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":313 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":313 * elif t == NPY_INT: f = "i" * elif t == NPY_UINT: f = "I" * elif t == NPY_LONG: f = "l" # <<<<<<<<<<<<<< @@ -3312,7 +3469,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_ULONG: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":314 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":314 * elif t == NPY_UINT: f = "I" * elif t == NPY_LONG: f = "l" * elif t == NPY_ULONG: f = "L" # <<<<<<<<<<<<<< @@ -3323,7 +3480,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_LONGLONG: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":315 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":315 * elif t == NPY_LONG: f = "l" * elif t == NPY_ULONG: f = "L" * elif t == NPY_LONGLONG: f = "q" # <<<<<<<<<<<<<< @@ -3334,7 +3491,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_ULONGLONG: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":316 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":316 * elif t == NPY_ULONG: f = "L" * elif t == NPY_LONGLONG: f = "q" * elif t == NPY_ULONGLONG: f = "Q" # <<<<<<<<<<<<<< @@ -3345,7 +3502,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_FLOAT: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":317 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":317 * elif t == NPY_LONGLONG: f = "q" * elif t == NPY_ULONGLONG: f = "Q" * elif t == NPY_FLOAT: f = "f" # <<<<<<<<<<<<<< @@ -3356,7 +3513,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_DOUBLE: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":318 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":318 * elif t == NPY_ULONGLONG: f = "Q" * elif t == NPY_FLOAT: f = "f" * elif t == NPY_DOUBLE: f = "d" # <<<<<<<<<<<<<< @@ -3367,7 +3524,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_LONGDOUBLE: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":319 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":319 * elif t == NPY_FLOAT: f = "f" * elif t == NPY_DOUBLE: f = "d" * elif t == NPY_LONGDOUBLE: f = "g" # <<<<<<<<<<<<<< @@ -3378,7 +3535,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_CFLOAT: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":320 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":320 * elif t == NPY_DOUBLE: f = "d" * elif t == NPY_LONGDOUBLE: f = "g" * elif t == NPY_CFLOAT: f = "Zf" # <<<<<<<<<<<<<< @@ -3389,7 +3546,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_CDOUBLE: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":321 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":321 * elif t == NPY_LONGDOUBLE: f = "g" * elif t == NPY_CFLOAT: f = "Zf" * elif t == NPY_CDOUBLE: f = "Zd" # <<<<<<<<<<<<<< @@ -3400,7 +3557,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_CLONGDOUBLE: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":322 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":322 * elif t == NPY_CFLOAT: f = "Zf" * elif t == NPY_CDOUBLE: f = "Zd" * elif t == NPY_CLONGDOUBLE: f = "Zg" # <<<<<<<<<<<<<< @@ -3411,7 +3568,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_OBJECT: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":323 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":323 * elif t == NPY_CDOUBLE: f = "Zd" * elif t == NPY_CLONGDOUBLE: f = "Zg" * elif t == NPY_OBJECT: f = "O" # <<<<<<<<<<<<<< @@ -3422,7 +3579,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; default: - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":325 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":325 * elif t == NPY_OBJECT: f = "O" * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) # <<<<<<<<<<<<<< @@ -3443,7 +3600,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":326 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":326 * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) * info.format = f # <<<<<<<<<<<<<< @@ -3452,7 +3609,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->format = __pyx_v_f; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":327 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":327 * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) * info.format = f * return # <<<<<<<<<<<<<< @@ -3462,7 +3619,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_r = 0; goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 * info.obj = self * * if not PyDataType_HASFIELDS(descr): # <<<<<<<<<<<<<< @@ -3471,7 +3628,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":329 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":329 * return * else: * info.format = PyObject_Malloc(_buffer_format_string_len) # <<<<<<<<<<<<<< @@ -3481,7 +3638,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P /*else*/ { __pyx_v_info->format = ((char *)PyObject_Malloc(0xFF)); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":330 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":330 * else: * info.format = PyObject_Malloc(_buffer_format_string_len) * info.format[0] = c'^' # Native data types, manual alignment # <<<<<<<<<<<<<< @@ -3490,7 +3647,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ (__pyx_v_info->format[0]) = '^'; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":331 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":331 * info.format = PyObject_Malloc(_buffer_format_string_len) * info.format[0] = c'^' # Native data types, manual alignment * offset = 0 # <<<<<<<<<<<<<< @@ -3499,7 +3656,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_offset = 0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":332 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":332 * info.format[0] = c'^' # Native data types, manual alignment * offset = 0 * f = _util_dtypestring(descr, info.format + 1, # <<<<<<<<<<<<<< @@ -3509,7 +3666,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_9 = __pyx_f_5numpy__util_dtypestring(__pyx_v_descr, (__pyx_v_info->format + 1), (__pyx_v_info->format + 0xFF), (&__pyx_v_offset)); if (unlikely(__pyx_t_9 == ((char *)NULL))) __PYX_ERR(1, 332, __pyx_L1_error) __pyx_v_f = __pyx_t_9; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":335 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":335 * info.format + _buffer_format_string_len, * &offset) * f[0] = c'\0' # Terminate format string # <<<<<<<<<<<<<< @@ -3519,7 +3676,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P (__pyx_v_f[0]) = '\x00'; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 * # experimental exception made for __getbuffer__ and __releasebuffer__ * # -- the details of this may change. * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< @@ -3551,7 +3708,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 * f[0] = c'\0' # Terminate format string * * def __releasebuffer__(ndarray self, Py_buffer* info): # <<<<<<<<<<<<<< @@ -3575,7 +3732,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s int __pyx_t_1; __Pyx_RefNannySetupContext("__releasebuffer__", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 * * def __releasebuffer__(ndarray self, Py_buffer* info): * if PyArray_HASFIELDS(self): # <<<<<<<<<<<<<< @@ -3585,7 +3742,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s __pyx_t_1 = (PyArray_HASFIELDS(__pyx_v_self) != 0); if (__pyx_t_1) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":339 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":339 * def __releasebuffer__(ndarray self, Py_buffer* info): * if PyArray_HASFIELDS(self): * PyObject_Free(info.format) # <<<<<<<<<<<<<< @@ -3594,7 +3751,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s */ PyObject_Free(__pyx_v_info->format); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 * * def __releasebuffer__(ndarray self, Py_buffer* info): * if PyArray_HASFIELDS(self): # <<<<<<<<<<<<<< @@ -3603,7 +3760,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 * if PyArray_HASFIELDS(self): * PyObject_Free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< @@ -3613,7 +3770,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s __pyx_t_1 = (((sizeof(npy_intp)) != (sizeof(Py_ssize_t))) != 0); if (__pyx_t_1) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":341 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":341 * PyObject_Free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): * PyObject_Free(info.strides) # <<<<<<<<<<<<<< @@ -3622,7 +3779,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s */ PyObject_Free(__pyx_v_info->strides); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 * if PyArray_HASFIELDS(self): * PyObject_Free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< @@ -3631,7 +3788,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 * f[0] = c'\0' # Terminate format string * * def __releasebuffer__(ndarray self, Py_buffer* info): # <<<<<<<<<<<<<< @@ -3643,7 +3800,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s __Pyx_RefNannyFinishContext(); } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 * ctypedef npy_cdouble complex_t * * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< @@ -3657,7 +3814,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew1", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":822 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":822 * * cdef inline object PyArray_MultiIterNew1(a): * return PyArray_MultiIterNew(1, a) # <<<<<<<<<<<<<< @@ -3671,7 +3828,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 * ctypedef npy_cdouble complex_t * * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< @@ -3690,7 +3847,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__ return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 * return PyArray_MultiIterNew(1, a) * * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< @@ -3704,7 +3861,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew2", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":825 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":825 * * cdef inline object PyArray_MultiIterNew2(a, b): * return PyArray_MultiIterNew(2, a, b) # <<<<<<<<<<<<<< @@ -3718,7 +3875,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 * return PyArray_MultiIterNew(1, a) * * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< @@ -3737,7 +3894,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__ return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":827 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":827 * return PyArray_MultiIterNew(2, a, b) * * cdef inline object PyArray_MultiIterNew3(a, b, c): # <<<<<<<<<<<<<< @@ -3751,7 +3908,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew3(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew3", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":828 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":828 * * cdef inline object PyArray_MultiIterNew3(a, b, c): * return PyArray_MultiIterNew(3, a, b, c) # <<<<<<<<<<<<<< @@ -3765,7 +3922,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew3(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":827 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":827 * return PyArray_MultiIterNew(2, a, b) * * cdef inline object PyArray_MultiIterNew3(a, b, c): # <<<<<<<<<<<<<< @@ -3784,7 +3941,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew3(PyObject *__ return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":830 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":830 * return PyArray_MultiIterNew(3, a, b, c) * * cdef inline object PyArray_MultiIterNew4(a, b, c, d): # <<<<<<<<<<<<<< @@ -3798,7 +3955,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew4(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew4", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":831 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":831 * * cdef inline object PyArray_MultiIterNew4(a, b, c, d): * return PyArray_MultiIterNew(4, a, b, c, d) # <<<<<<<<<<<<<< @@ -3812,7 +3969,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew4(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":830 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":830 * return PyArray_MultiIterNew(3, a, b, c) * * cdef inline object PyArray_MultiIterNew4(a, b, c, d): # <<<<<<<<<<<<<< @@ -3831,7 +3988,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew4(PyObject *__ return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":833 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":833 * return PyArray_MultiIterNew(4, a, b, c, d) * * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): # <<<<<<<<<<<<<< @@ -3845,7 +4002,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew5(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew5", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":834 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":834 * * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): * return PyArray_MultiIterNew(5, a, b, c, d, e) # <<<<<<<<<<<<<< @@ -3859,7 +4016,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew5(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":833 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":833 * return PyArray_MultiIterNew(4, a, b, c, d) * * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): # <<<<<<<<<<<<<< @@ -3878,7 +4035,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew5(PyObject *__ return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":836 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":836 * return PyArray_MultiIterNew(5, a, b, c, d, e) * * cdef inline tuple PyDataType_SHAPE(dtype d): # <<<<<<<<<<<<<< @@ -3892,7 +4049,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ int __pyx_t_1; __Pyx_RefNannySetupContext("PyDataType_SHAPE", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":837 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":837 * * cdef inline tuple PyDataType_SHAPE(dtype d): * if PyDataType_HASSUBARRAY(d): # <<<<<<<<<<<<<< @@ -3902,7 +4059,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ __pyx_t_1 = (PyDataType_HASSUBARRAY(__pyx_v_d) != 0); if (__pyx_t_1) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":838 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":838 * cdef inline tuple PyDataType_SHAPE(dtype d): * if PyDataType_HASSUBARRAY(d): * return d.subarray.shape # <<<<<<<<<<<<<< @@ -3914,7 +4071,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ __pyx_r = ((PyObject*)__pyx_v_d->subarray->shape); goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":837 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":837 * * cdef inline tuple PyDataType_SHAPE(dtype d): * if PyDataType_HASSUBARRAY(d): # <<<<<<<<<<<<<< @@ -3923,7 +4080,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":840 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":840 * return d.subarray.shape * else: * return () # <<<<<<<<<<<<<< @@ -3937,7 +4094,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ goto __pyx_L0; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":836 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":836 * return PyArray_MultiIterNew(5, a, b, c, d, e) * * cdef inline tuple PyDataType_SHAPE(dtype d): # <<<<<<<<<<<<<< @@ -3952,7 +4109,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":842 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":842 * return () * * cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset) except NULL: # <<<<<<<<<<<<<< @@ -3981,7 +4138,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx char *__pyx_t_9; __Pyx_RefNannySetupContext("_util_dtypestring", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":847 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":847 * * cdef dtype child * cdef int endian_detector = 1 # <<<<<<<<<<<<<< @@ -3990,7 +4147,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ __pyx_v_endian_detector = 1; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":848 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":848 * cdef dtype child * cdef int endian_detector = 1 * cdef bint little_endian = ((&endian_detector)[0] != 0) # <<<<<<<<<<<<<< @@ -3999,7 +4156,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ __pyx_v_little_endian = ((((char *)(&__pyx_v_endian_detector))[0]) != 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":851 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":851 * cdef tuple fields * * for childname in descr.names: # <<<<<<<<<<<<<< @@ -4022,7 +4179,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_XDECREF_SET(__pyx_v_childname, __pyx_t_3); __pyx_t_3 = 0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":852 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":852 * * for childname in descr.names: * fields = descr.fields[childname] # <<<<<<<<<<<<<< @@ -4039,7 +4196,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_XDECREF_SET(__pyx_v_fields, ((PyObject*)__pyx_t_3)); __pyx_t_3 = 0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":853 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":853 * for childname in descr.names: * fields = descr.fields[childname] * child, new_offset = fields # <<<<<<<<<<<<<< @@ -4074,7 +4231,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_XDECREF_SET(__pyx_v_new_offset, __pyx_t_4); __pyx_t_4 = 0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":855 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":855 * child, new_offset = fields * * if (end - f) - (new_offset - offset[0]) < 15: # <<<<<<<<<<<<<< @@ -4091,7 +4248,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_6 = ((((__pyx_v_end - __pyx_v_f) - ((int)__pyx_t_5)) < 15) != 0); if (unlikely(__pyx_t_6)) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":856 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":856 * * if (end - f) - (new_offset - offset[0]) < 15: * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") # <<<<<<<<<<<<<< @@ -4104,7 +4261,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 856, __pyx_L1_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":855 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":855 * child, new_offset = fields * * if (end - f) - (new_offset - offset[0]) < 15: # <<<<<<<<<<<<<< @@ -4113,7 +4270,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * * if ((child.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -4133,7 +4290,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx } __pyx_L8_next_or:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":859 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":859 * * if ((child.byteorder == c'>' and little_endian) or * (child.byteorder == c'<' and not little_endian)): # <<<<<<<<<<<<<< @@ -4150,7 +4307,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_6 = __pyx_t_7; __pyx_L7_bool_binop_done:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * * if ((child.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -4159,7 +4316,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ if (unlikely(__pyx_t_6)) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":860 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":860 * if ((child.byteorder == c'>' and little_endian) or * (child.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< @@ -4172,7 +4329,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 860, __pyx_L1_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * * if ((child.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -4181,7 +4338,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":870 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":870 * * # Output padding bytes * while offset[0] < new_offset: # <<<<<<<<<<<<<< @@ -4197,7 +4354,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (!__pyx_t_6) break; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":871 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":871 * # Output padding bytes * while offset[0] < new_offset: * f[0] = 120 # "x"; pad byte # <<<<<<<<<<<<<< @@ -4206,7 +4363,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ (__pyx_v_f[0]) = 0x78; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":872 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":872 * while offset[0] < new_offset: * f[0] = 120 # "x"; pad byte * f += 1 # <<<<<<<<<<<<<< @@ -4215,7 +4372,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ __pyx_v_f = (__pyx_v_f + 1); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":873 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":873 * f[0] = 120 # "x"; pad byte * f += 1 * offset[0] += 1 # <<<<<<<<<<<<<< @@ -4226,7 +4383,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx (__pyx_v_offset[__pyx_t_8]) = ((__pyx_v_offset[__pyx_t_8]) + 1); } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":875 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":875 * offset[0] += 1 * * offset[0] += child.itemsize # <<<<<<<<<<<<<< @@ -4236,7 +4393,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_8 = 0; (__pyx_v_offset[__pyx_t_8]) = ((__pyx_v_offset[__pyx_t_8]) + __pyx_v_child->elsize); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":877 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":877 * offset[0] += child.itemsize * * if not PyDataType_HASFIELDS(child): # <<<<<<<<<<<<<< @@ -4246,7 +4403,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_6 = ((!(PyDataType_HASFIELDS(__pyx_v_child) != 0)) != 0); if (__pyx_t_6) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":878 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":878 * * if not PyDataType_HASFIELDS(child): * t = child.type_num # <<<<<<<<<<<<<< @@ -4258,7 +4415,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_XDECREF_SET(__pyx_v_t, __pyx_t_4); __pyx_t_4 = 0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":879 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":879 * if not PyDataType_HASFIELDS(child): * t = child.type_num * if end - f < 5: # <<<<<<<<<<<<<< @@ -4268,7 +4425,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_6 = (((__pyx_v_end - __pyx_v_f) < 5) != 0); if (unlikely(__pyx_t_6)) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":880 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":880 * t = child.type_num * if end - f < 5: * raise RuntimeError(u"Format string allocated too short.") # <<<<<<<<<<<<<< @@ -4281,7 +4438,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __PYX_ERR(1, 880, __pyx_L1_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":879 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":879 * if not PyDataType_HASFIELDS(child): * t = child.type_num * if end - f < 5: # <<<<<<<<<<<<<< @@ -4290,7 +4447,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":883 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":883 * * # Until ticket #99 is fixed, use integers to avoid warnings * if t == NPY_BYTE: f[0] = 98 #"b" # <<<<<<<<<<<<<< @@ -4308,7 +4465,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":884 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":884 * # Until ticket #99 is fixed, use integers to avoid warnings * if t == NPY_BYTE: f[0] = 98 #"b" * elif t == NPY_UBYTE: f[0] = 66 #"B" # <<<<<<<<<<<<<< @@ -4326,7 +4483,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":885 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":885 * if t == NPY_BYTE: f[0] = 98 #"b" * elif t == NPY_UBYTE: f[0] = 66 #"B" * elif t == NPY_SHORT: f[0] = 104 #"h" # <<<<<<<<<<<<<< @@ -4344,7 +4501,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":886 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":886 * elif t == NPY_UBYTE: f[0] = 66 #"B" * elif t == NPY_SHORT: f[0] = 104 #"h" * elif t == NPY_USHORT: f[0] = 72 #"H" # <<<<<<<<<<<<<< @@ -4362,7 +4519,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":887 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":887 * elif t == NPY_SHORT: f[0] = 104 #"h" * elif t == NPY_USHORT: f[0] = 72 #"H" * elif t == NPY_INT: f[0] = 105 #"i" # <<<<<<<<<<<<<< @@ -4380,7 +4537,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":888 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":888 * elif t == NPY_USHORT: f[0] = 72 #"H" * elif t == NPY_INT: f[0] = 105 #"i" * elif t == NPY_UINT: f[0] = 73 #"I" # <<<<<<<<<<<<<< @@ -4398,7 +4555,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":889 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":889 * elif t == NPY_INT: f[0] = 105 #"i" * elif t == NPY_UINT: f[0] = 73 #"I" * elif t == NPY_LONG: f[0] = 108 #"l" # <<<<<<<<<<<<<< @@ -4416,7 +4573,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":890 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":890 * elif t == NPY_UINT: f[0] = 73 #"I" * elif t == NPY_LONG: f[0] = 108 #"l" * elif t == NPY_ULONG: f[0] = 76 #"L" # <<<<<<<<<<<<<< @@ -4434,7 +4591,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":891 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":891 * elif t == NPY_LONG: f[0] = 108 #"l" * elif t == NPY_ULONG: f[0] = 76 #"L" * elif t == NPY_LONGLONG: f[0] = 113 #"q" # <<<<<<<<<<<<<< @@ -4452,7 +4609,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":892 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":892 * elif t == NPY_ULONG: f[0] = 76 #"L" * elif t == NPY_LONGLONG: f[0] = 113 #"q" * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" # <<<<<<<<<<<<<< @@ -4470,7 +4627,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":893 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":893 * elif t == NPY_LONGLONG: f[0] = 113 #"q" * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" * elif t == NPY_FLOAT: f[0] = 102 #"f" # <<<<<<<<<<<<<< @@ -4488,7 +4645,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":894 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":894 * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" * elif t == NPY_FLOAT: f[0] = 102 #"f" * elif t == NPY_DOUBLE: f[0] = 100 #"d" # <<<<<<<<<<<<<< @@ -4506,7 +4663,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":895 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":895 * elif t == NPY_FLOAT: f[0] = 102 #"f" * elif t == NPY_DOUBLE: f[0] = 100 #"d" * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" # <<<<<<<<<<<<<< @@ -4524,7 +4681,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":896 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":896 * elif t == NPY_DOUBLE: f[0] = 100 #"d" * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf # <<<<<<<<<<<<<< @@ -4544,7 +4701,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":897 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":897 * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd # <<<<<<<<<<<<<< @@ -4564,7 +4721,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":898 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":898 * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd * elif t == NPY_CLONGDOUBLE: f[0] = 90; f[1] = 103; f += 1 # Zg # <<<<<<<<<<<<<< @@ -4584,7 +4741,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":899 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":899 * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd * elif t == NPY_CLONGDOUBLE: f[0] = 90; f[1] = 103; f += 1 # Zg * elif t == NPY_OBJECT: f[0] = 79 #"O" # <<<<<<<<<<<<<< @@ -4602,7 +4759,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":901 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":901 * elif t == NPY_OBJECT: f[0] = 79 #"O" * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) # <<<<<<<<<<<<<< @@ -4621,7 +4778,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx } __pyx_L15:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":902 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":902 * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) * f += 1 # <<<<<<<<<<<<<< @@ -4630,7 +4787,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ __pyx_v_f = (__pyx_v_f + 1); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":877 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":877 * offset[0] += child.itemsize * * if not PyDataType_HASFIELDS(child): # <<<<<<<<<<<<<< @@ -4640,7 +4797,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L13; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":906 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":906 * # Cython ignores struct boundary information ("T{...}"), * # so don't output it * f = _util_dtypestring(child, f, end, offset) # <<<<<<<<<<<<<< @@ -4653,7 +4810,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx } __pyx_L13:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":851 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":851 * cdef tuple fields * * for childname in descr.names: # <<<<<<<<<<<<<< @@ -4663,7 +4820,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":907 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":907 * # so don't output it * f = _util_dtypestring(child, f, end, offset) * return f # <<<<<<<<<<<<<< @@ -4673,7 +4830,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_r = __pyx_v_f; goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":842 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":842 * return () * * cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset) except NULL: # <<<<<<<<<<<<<< @@ -4698,7 +4855,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1022 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1022 * int _import_umath() except -1 * * cdef inline void set_array_base(ndarray arr, object base): # <<<<<<<<<<<<<< @@ -4710,7 +4867,7 @@ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_a __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("set_array_base", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1023 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1023 * * cdef inline void set_array_base(ndarray arr, object base): * Py_INCREF(base) # important to do this before stealing the reference below! # <<<<<<<<<<<<<< @@ -4719,7 +4876,7 @@ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_a */ Py_INCREF(__pyx_v_base); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1024 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1024 * cdef inline void set_array_base(ndarray arr, object base): * Py_INCREF(base) # important to do this before stealing the reference below! * PyArray_SetBaseObject(arr, base) # <<<<<<<<<<<<<< @@ -4728,7 +4885,7 @@ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_a */ (void)(PyArray_SetBaseObject(__pyx_v_arr, __pyx_v_base)); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1022 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1022 * int _import_umath() except -1 * * cdef inline void set_array_base(ndarray arr, object base): # <<<<<<<<<<<<<< @@ -4740,7 +4897,7 @@ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_a __Pyx_RefNannyFinishContext(); } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1026 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1026 * PyArray_SetBaseObject(arr, base) * * cdef inline object get_array_base(ndarray arr): # <<<<<<<<<<<<<< @@ -4755,7 +4912,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py int __pyx_t_1; __Pyx_RefNannySetupContext("get_array_base", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1027 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1027 * * cdef inline object get_array_base(ndarray arr): * base = PyArray_BASE(arr) # <<<<<<<<<<<<<< @@ -4764,7 +4921,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py */ __pyx_v_base = PyArray_BASE(__pyx_v_arr); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1028 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1028 * cdef inline object get_array_base(ndarray arr): * base = PyArray_BASE(arr) * if base is NULL: # <<<<<<<<<<<<<< @@ -4774,7 +4931,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py __pyx_t_1 = ((__pyx_v_base == NULL) != 0); if (__pyx_t_1) { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1029 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1029 * base = PyArray_BASE(arr) * if base is NULL: * return None # <<<<<<<<<<<<<< @@ -4785,7 +4942,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1028 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1028 * cdef inline object get_array_base(ndarray arr): * base = PyArray_BASE(arr) * if base is NULL: # <<<<<<<<<<<<<< @@ -4794,7 +4951,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py */ } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1030 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1030 * if base is NULL: * return None * return base # <<<<<<<<<<<<<< @@ -4806,7 +4963,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py __pyx_r = ((PyObject *)__pyx_v_base); goto __pyx_L0; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1026 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1026 * PyArray_SetBaseObject(arr, base) * * cdef inline object get_array_base(ndarray arr): # <<<<<<<<<<<<<< @@ -4821,7 +4978,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1034 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1034 * # Versions of the import_* functions which are more suitable for * # Cython code. * cdef inline int import_array() except -1: # <<<<<<<<<<<<<< @@ -4842,7 +4999,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { PyObject *__pyx_t_8 = NULL; __Pyx_RefNannySetupContext("import_array", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 * # Cython code. * cdef inline int import_array() except -1: * try: # <<<<<<<<<<<<<< @@ -4858,7 +5015,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { __Pyx_XGOTREF(__pyx_t_3); /*try:*/ { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1036 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1036 * cdef inline int import_array() except -1: * try: * _import_array() # <<<<<<<<<<<<<< @@ -4867,7 +5024,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { */ __pyx_t_4 = _import_array(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 1036, __pyx_L3_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 * # Cython code. * cdef inline int import_array() except -1: * try: # <<<<<<<<<<<<<< @@ -4881,7 +5038,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { goto __pyx_L8_try_end; __pyx_L3_error:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1037 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1037 * try: * _import_array() * except Exception: # <<<<<<<<<<<<<< @@ -4896,7 +5053,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { __Pyx_GOTREF(__pyx_t_6); __Pyx_GOTREF(__pyx_t_7); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1038 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1038 * _import_array() * except Exception: * raise ImportError("numpy.core.multiarray failed to import") # <<<<<<<<<<<<<< @@ -4912,7 +5069,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 * # Cython code. * cdef inline int import_array() except -1: * try: # <<<<<<<<<<<<<< @@ -4927,7 +5084,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { __pyx_L8_try_end:; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1034 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1034 * # Versions of the import_* functions which are more suitable for * # Cython code. * cdef inline int import_array() except -1: # <<<<<<<<<<<<<< @@ -4950,7 +5107,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1040 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1040 * raise ImportError("numpy.core.multiarray failed to import") * * cdef inline int import_umath() except -1: # <<<<<<<<<<<<<< @@ -4971,7 +5128,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { PyObject *__pyx_t_8 = NULL; __Pyx_RefNannySetupContext("import_umath", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 * * cdef inline int import_umath() except -1: * try: # <<<<<<<<<<<<<< @@ -4987,7 +5144,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { __Pyx_XGOTREF(__pyx_t_3); /*try:*/ { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1042 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1042 * cdef inline int import_umath() except -1: * try: * _import_umath() # <<<<<<<<<<<<<< @@ -4996,7 +5153,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { */ __pyx_t_4 = _import_umath(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 1042, __pyx_L3_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 * * cdef inline int import_umath() except -1: * try: # <<<<<<<<<<<<<< @@ -5010,7 +5167,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { goto __pyx_L8_try_end; __pyx_L3_error:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1043 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1043 * try: * _import_umath() * except Exception: # <<<<<<<<<<<<<< @@ -5025,7 +5182,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { __Pyx_GOTREF(__pyx_t_6); __Pyx_GOTREF(__pyx_t_7); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1044 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1044 * _import_umath() * except Exception: * raise ImportError("numpy.core.umath failed to import") # <<<<<<<<<<<<<< @@ -5041,7 +5198,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 * * cdef inline int import_umath() except -1: * try: # <<<<<<<<<<<<<< @@ -5056,7 +5213,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { __pyx_L8_try_end:; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1040 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1040 * raise ImportError("numpy.core.multiarray failed to import") * * cdef inline int import_umath() except -1: # <<<<<<<<<<<<<< @@ -5079,7 +5236,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { return __pyx_r; } -/* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 * raise ImportError("numpy.core.umath failed to import") * * cdef inline int import_ufunc() except -1: # <<<<<<<<<<<<<< @@ -5100,7 +5257,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { PyObject *__pyx_t_8 = NULL; __Pyx_RefNannySetupContext("import_ufunc", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 * * cdef inline int import_ufunc() except -1: * try: # <<<<<<<<<<<<<< @@ -5116,7 +5273,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { __Pyx_XGOTREF(__pyx_t_3); /*try:*/ { - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1048 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1048 * cdef inline int import_ufunc() except -1: * try: * _import_umath() # <<<<<<<<<<<<<< @@ -5125,7 +5282,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { */ __pyx_t_4 = _import_umath(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 1048, __pyx_L3_error) - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 * * cdef inline int import_ufunc() except -1: * try: # <<<<<<<<<<<<<< @@ -5139,7 +5296,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { goto __pyx_L8_try_end; __pyx_L3_error:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1049 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1049 * try: * _import_umath() * except Exception: # <<<<<<<<<<<<<< @@ -5153,7 +5310,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { __Pyx_GOTREF(__pyx_t_6); __Pyx_GOTREF(__pyx_t_7); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1050 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1050 * _import_umath() * except Exception: * raise ImportError("numpy.core.umath failed to import") # <<<<<<<<<<<<<< @@ -5167,7 +5324,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 * * cdef inline int import_ufunc() except -1: * try: # <<<<<<<<<<<<<< @@ -5182,7 +5339,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { __pyx_L8_try_end:; } - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 * raise ImportError("numpy.core.umath failed to import") * * cdef inline int import_ufunc() except -1: # <<<<<<<<<<<<<< @@ -5425,8 +5582,9 @@ static PyTypeObject __pyx_scope_struct____Pyx_CFunc_object____object___to_py = { static PyMethodDef __pyx_methods[] = { {"ft_hash", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_ft_hash}, - {"ft_hash_broken", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_2ft_hash_broken}, - {"compute_ngrams", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams, METH_VARARGS|METH_KEYWORDS, __pyx_doc_6gensim_6models_14_utils_any2vec_4compute_ngrams}, + {"ft_hash_bytes", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_bytes, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_2ft_hash_bytes}, + {"ft_hash_broken", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_5ft_hash_broken, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_4ft_hash_broken}, + {"compute_ngrams", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams, METH_VARARGS|METH_KEYWORDS, __pyx_doc_6gensim_6models_14_utils_any2vec_6compute_ngrams}, {0, 0, 0, 0} }; @@ -5509,7 +5667,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {0, 0, 0, 0, 0, 0, 0} }; static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(0, 91, __pyx_L1_error) + __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(0, 122, __pyx_L1_error) __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 272, __pyx_L1_error) __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_n_s_RuntimeError); if (!__pyx_builtin_RuntimeError) __PYX_ERR(1, 856, __pyx_L1_error) __pyx_builtin_ImportError = __Pyx_GetBuiltinName(__pyx_n_s_ImportError); if (!__pyx_builtin_ImportError) __PYX_ERR(1, 1038, __pyx_L1_error) @@ -5522,7 +5680,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): * raise ValueError(u"ndarray is not C contiguous") # <<<<<<<<<<<<<< @@ -5533,7 +5691,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__3); __Pyx_GIVEREF(__pyx_tuple__3); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): * raise ValueError(u"ndarray is not Fortran contiguous") # <<<<<<<<<<<<<< @@ -5544,7 +5702,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__4); __Pyx_GIVEREF(__pyx_tuple__4); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 * if ((descr.byteorder == c'>' and little_endian) or * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< @@ -5555,7 +5713,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__5); __Pyx_GIVEREF(__pyx_tuple__5); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":856 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":856 * * if (end - f) - (new_offset - offset[0]) < 15: * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") # <<<<<<<<<<<<<< @@ -5566,7 +5724,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__6); __Pyx_GIVEREF(__pyx_tuple__6); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":880 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":880 * t = child.type_num * if end - f < 5: * raise RuntimeError(u"Format string allocated too short.") # <<<<<<<<<<<<<< @@ -5577,7 +5735,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__7); __Pyx_GIVEREF(__pyx_tuple__7); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1038 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1038 * _import_array() * except Exception: * raise ImportError("numpy.core.multiarray failed to import") # <<<<<<<<<<<<<< @@ -5588,7 +5746,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__8); __Pyx_GIVEREF(__pyx_tuple__8); - /* "../../.virtualenvs/release/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1044 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1044 * _import_umath() * except Exception: * raise ImportError("numpy.core.umath failed to import") # <<<<<<<<<<<<<< @@ -5929,62 +6087,62 @@ if (!__Pyx_RefNanny) { if (__Pyx_patch_abc() < 0) __PYX_ERR(0, 1, __pyx_L1_error) #endif - /* "gensim/models/_utils_any2vec.pyx":10 - * """General functions used for any2vec models.""" + /* "gensim/models/_utils_any2vec.pyx":11 * + * from libc.stdint cimport uint32_t, int8_t * from six import PY2 # <<<<<<<<<<<<<< * import numpy as np * cimport numpy as np */ - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 10, __pyx_L1_error) + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 11, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_n_s_PY2); __Pyx_GIVEREF(__pyx_n_s_PY2); PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_PY2); - __pyx_t_2 = __Pyx_Import(__pyx_n_s_six, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 10, __pyx_L1_error) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_six, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 11, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 10, __pyx_L1_error) + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 11, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_PY2, __pyx_t_1) < 0) __PYX_ERR(0, 10, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_PY2, __pyx_t_1) < 0) __PYX_ERR(0, 11, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "gensim/models/_utils_any2vec.pyx":11 - * + /* "gensim/models/_utils_any2vec.pyx":12 + * from libc.stdint cimport uint32_t, int8_t * from six import PY2 * import numpy as np # <<<<<<<<<<<<<< * cimport numpy as np * */ - __pyx_t_2 = __Pyx_Import(__pyx_n_s_numpy, 0, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 11, __pyx_L1_error) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_numpy, 0, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 12, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_np, __pyx_t_2) < 0) __PYX_ERR(0, 11, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_np, __pyx_t_2) < 0) __PYX_ERR(0, 12, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "gensim/models/_utils_any2vec.pyx":21 + /* "gensim/models/_utils_any2vec.pyx":22 * return ord(b) * * _byte_to_int = _byte_to_int_py2 if PY2 else _byte_to_int_py3 # <<<<<<<<<<<<<< * * */ - __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 21, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 21, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_3) { - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 21, __pyx_L1_error) + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __pyx_t_1; __pyx_t_1 = 0; } else { - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 21, __pyx_L1_error) + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __pyx_t_1; __pyx_t_1 = 0; } - if (PyDict_SetItem(__pyx_d, __pyx_n_s_byte_to_int, __pyx_t_2) < 0) __PYX_ERR(0, 21, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_byte_to_int, __pyx_t_2) < 0) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; /* "gensim/models/_utils_any2vec.pyx":1 @@ -8310,6 +8468,37 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value) } } +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint32_t(uint32_t value) { + const uint32_t neg_one = (uint32_t) ((uint32_t) 0 - (uint32_t) 1), const_zero = (uint32_t) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(uint32_t) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(uint32_t) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(uint32_t) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(uint32_t) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(uint32_t) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(uint32_t), + little, !is_unsigned); + } +} + /* Declarations */ #if CYTHON_CCOMPLEX #ifdef __cplusplus @@ -8871,6 +9060,195 @@ static CYTHON_INLINE unsigned int __Pyx_PyInt_As_unsigned_int(PyObject *x) { return (unsigned int) -1; } +/* CIntFromPy */ +static CYTHON_INLINE char __Pyx_PyInt_As_char(PyObject *x) { + const char neg_one = (char) ((char) 0 - (char) 1), const_zero = (char) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(char) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(char, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (char) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (char) 0; + case 1: __PYX_VERIFY_RETURN_INT(char, digit, digits[0]) + case 2: + if (8 * sizeof(char) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) >= 2 * PyLong_SHIFT) { + return (char) (((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(char) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) >= 3 * PyLong_SHIFT) { + return (char) (((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(char) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) >= 4 * PyLong_SHIFT) { + return (char) (((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (char) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(char) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(char, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(char) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(char, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (char) 0; + case -1: __PYX_VERIFY_RETURN_INT(char, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(char, digit, +digits[0]) + case -2: + if (8 * sizeof(char) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) - 1 > 2 * PyLong_SHIFT) { + return (char) (((char)-1)*(((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(char) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) - 1 > 2 * PyLong_SHIFT) { + return (char) ((((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(char) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) - 1 > 3 * PyLong_SHIFT) { + return (char) (((char)-1)*(((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(char) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) - 1 > 3 * PyLong_SHIFT) { + return (char) ((((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(char) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) - 1 > 4 * PyLong_SHIFT) { + return (char) (((char)-1)*(((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(char) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) - 1 > 4 * PyLong_SHIFT) { + return (char) ((((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + } +#endif + if (sizeof(char) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(char, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(char) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(char, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + char val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (char) -1; + } + } else { + char val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (char) -1; + val = __Pyx_PyInt_As_char(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to char"); + return (char) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to char"); + return (char) -1; +} + /* CIntFromPy */ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { const int neg_one = (int) ((int) 0 - (int) 1), const_zero = (int) 0; diff --git a/gensim/models/_utils_any2vec.pyx b/gensim/models/_utils_any2vec.pyx index f27d7e400f..f5672c0b14 100644 --- a/gensim/models/_utils_any2vec.pyx +++ b/gensim/models/_utils_any2vec.pyx @@ -7,6 +7,7 @@ """General functions used for any2vec models.""" +from libc.stdint cimport uint32_t, int8_t from six import PY2 import numpy as np cimport numpy as np @@ -26,6 +27,11 @@ cpdef ft_hash(unicode string): Reproduce `hash method from Facebook fastText implementation `_. + This function is expensive because it encodes each input string as UTF-8, + and this can accumulate to a high computational cost if there are many + strings. If possible, convert your strings to bytes and use the faster + :py:func:`ft_hash_bytes` function instead. + Parameters ---------- string : unicode @@ -44,6 +50,31 @@ cpdef ft_hash(unicode string): return h +cpdef ft_hash_bytes(bytes bytez): + """Calculate hash based on `bytez`. + Reproduce `hash method from Facebook fastText implementation + `_. + + Parameters + ---------- + bytez : bytes + The string whose hash needs to be calculated, encoded as UTF-8. + + Returns + ------- + unsigned int + The hash of the string. + + """ + cdef uint32_t h = 2166136261 + cdef char b + + for b in bytez: + h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss + h = h * 16777619 + return h + + cpdef ft_hash_broken(unicode string): """Calculate hash based on `string`. diff --git a/gensim/models/fasttext.py b/gensim/models/fasttext.py index f25ff4481d..b51494578c 100644 --- a/gensim/models/fasttext.py +++ b/gensim/models/fasttext.py @@ -98,7 +98,7 @@ from gensim.models.word2vec import Word2VecVocab, Word2VecTrainables, train_sg_pair, train_cbow_pair from gensim.models.keyedvectors import FastTextKeyedVectors from gensim.models.base_any2vec import BaseWordEmbeddingsModel -from gensim.models.utils_any2vec import _compute_ngrams, _ft_hash, _ft_hash_broken +from gensim.models.utils_any2vec import ft_ngram_hashes, ft_ngram_hashes_broken from smart_open import smart_open from gensim.utils import deprecated, call_on_class_only @@ -554,7 +554,7 @@ def _clear_post_train(self): self.wv.buckets_word = None def estimate_memory(self, vocab_size=None, report=None): - hash_fn = _ft_hash if self.wv.compatible_hash else _ft_hash_broken + hash_fn = ft_ngram_hashes if self.wv.compatible_hash else ft_ngram_hashes_broken vocab_size = vocab_size or len(self.wv.vocab) vec_size = self.vector_size * np.dtype(np.float32).itemsize @@ -574,9 +574,9 @@ def estimate_memory(self, vocab_size=None, report=None): buckets = set() num_ngrams = 0 for word in self.wv.vocab: - ngrams = _compute_ngrams(word, self.wv.min_n, self.wv.max_n) - num_ngrams += len(ngrams) - buckets.update(hash_fn(ng) % self.trainables.bucket for ng in ngrams) + hashes = hash_fn(word, self.wv.min_n, self.wv.max_n, self.trainables.bucket) + num_ngrams += len(hashes) + buckets.update(hashes) num_buckets = len(buckets) report['syn0_ngrams'] = num_buckets * vec_size # A tuple (48 bytes) with num_ngrams_word ints (8 bytes) for each word diff --git a/gensim/models/keyedvectors.py b/gensim/models/keyedvectors.py index d9dad1cc56..26d6ecbf98 100644 --- a/gensim/models/keyedvectors.py +++ b/gensim/models/keyedvectors.py @@ -182,9 +182,8 @@ from gensim.models.utils_any2vec import ( _save_word2vec_format, _load_word2vec_format, - _compute_ngrams, - _ft_hash, - _ft_hash_broken + ft_ngram_hashes, + ft_ngram_hashes_broken, ) from gensim.similarities.termsim import TermSimilarityIndex, SparseTermSimilarityMatrix @@ -2012,9 +2011,9 @@ def __contains__(self, word): if word in self.vocab: return True else: - hash_fn = _ft_hash if self.compatible_hash else _ft_hash_broken - char_ngrams = _compute_ngrams(word, self.min_n, self.max_n) - return any(hash_fn(ng) % self.bucket in self.hash2index for ng in char_ngrams) + hash_fn = ft_ngram_hashes if self.compatible_hash else ft_ngram_hashes_broken + hashes = hash_fn(word, self.min_n, self.max_n, self.bucket) + return any(h in self.hash2index for h in hashes) def save(self, *args, **kwargs): """Save object. @@ -2056,23 +2055,20 @@ def word_vec(self, word, use_norm=False): If word and all ngrams not in vocabulary. """ - hash_fn = _ft_hash if self.compatible_hash else _ft_hash_broken + hash_fn = ft_ngram_hashes if self.compatible_hash else ft_ngram_hashes_broken if word in self.vocab: return super(FastTextKeyedVectors, self).word_vec(word, use_norm) elif self.bucket == 0: raise KeyError('cannot calculate vector for OOV word without ngrams') else: - # from gensim.models.fasttext import compute_ngrams word_vec = np.zeros(self.vectors_ngrams.shape[1], dtype=np.float32) - ngrams = _compute_ngrams(word, self.min_n, self.max_n) if use_norm: ngram_weights = self.vectors_ngrams_norm else: ngram_weights = self.vectors_ngrams ngrams_found = 0 - for ngram in ngrams: - ngram_hash = hash_fn(ngram) % self.bucket + for ngram_hash in hash_fn(word, self.min_n, self.max_n, self.bucket): if ngram_hash in self.hash2index: word_vec += ngram_weights[self.hash2index[ngram_hash]] ngrams_found += 1 @@ -2130,7 +2126,7 @@ def init_ngrams_weights(self, seed): self.min_n, self.max_n, self.bucket, - _ft_hash if self.compatible_hash else _ft_hash_broken, + ft_ngram_hashes if self.compatible_hash else ft_ngram_hashes_broken, self.hash2index ) self.num_ngram_vectors = len(ngram_indices) @@ -2153,7 +2149,7 @@ def update_ngrams_weights(self, seed, old_vocab_len): self.min_n, self.max_n, self.bucket, - _ft_hash if self.compatible_hash else _ft_hash_broken, + ft_ngram_hashes if self.compatible_hash else ft_ngram_hashes_broken, self.hash2index ) num_new_ngrams = len(new_ngram_hashes) @@ -2215,7 +2211,7 @@ def init_post_load(self, vectors, match_gensim=False): self.min_n, self.max_n, self.bucket, - _ft_hash if self.compatible_hash else _ft_hash_broken, + ft_ngram_hashes if self.compatible_hash else ft_ngram_hashes_broken, dict(), # we don't care what goes here in this case ) ngram_hashes = sorted(set(ngram_hashes)) @@ -2237,15 +2233,14 @@ def adjust_vectors(self): if self.bucket == 0: return - hash_fn = _ft_hash if self.compatible_hash else _ft_hash_broken + hash_fn = ft_ngram_hashes if self.compatible_hash else ft_ngram_hashes_broken for w, v in self.vocab.items(): word_vec = np.copy(self.vectors_vocab[v.index]) - ngrams = _compute_ngrams(w, self.min_n, self.max_n) - for ngram in ngrams: - ngram_index = self.hash2index[hash_fn(ngram) % self.bucket] - word_vec += self.vectors_ngrams[ngram_index] - word_vec /= len(ngrams) + 1 + ngram_hashes = hash_fn(w, self.min_n, self.max_n, self.bucket) + for nh in ngram_hashes: + word_vec += self.vectors_ngrams[self.hash2index[nh]] + word_vec /= len(ngram_hashes) + 1 self.vectors[v.index] = word_vec @@ -2293,8 +2288,7 @@ def _process_fasttext_vocab(iterable, min_n, max_n, num_buckets, hash_fn, hash2i for word, vocab in iterable: wi = [] - for ngram in _compute_ngrams(word, min_n, max_n): - ngram_hash = hash_fn(ngram) % num_buckets + for ngram_hash in hash_fn(word, min_n, max_n, num_buckets): if ngram_hash not in hash2index: # # This is a new ngram. Reserve a new index in hash2index. diff --git a/gensim/models/utils_any2vec.py b/gensim/models/utils_any2vec.py index 74d0effcff..7b6e9a499a 100644 --- a/gensim/models/utils_any2vec.py +++ b/gensim/models/utils_any2vec.py @@ -4,7 +4,21 @@ # Author: Shiva Manne # Copyright (C) 2018 RaRe Technologies s.r.o. -"""General functions used for any2vec models.""" +"""General functions used for any2vec models. + +One of the goals of this module is to provide an abstraction over the Cython +extensions for FastText. If they are not available, then the module substitutes +slower Python versions in their place. + +Another related set of FastText functionality is computing ngrams for a word, +and then hashing them. The :py:func:`ft_ngram_hashes` and +:py:func:`ft_ngram_hashes_broken` achieves this goal. + +Finally, the module also exposes "working" and "broken" hash functions for +FastText. It does this without abstracting them away, because the correct +function to use depends on the current model. + +""" import logging import numpy as np @@ -30,18 +44,18 @@ def _byte_to_int_py2(b): # -# Define this here so we can unittest here. Only use this function if the -# faster C version fails to import. +# Define this here so we can unittest this function directly. +# Only use this function if the faster C version fails to import. # -def _ft_hash_py(string): - """Calculate hash based on `string`. +def _ft_hash_py_bytes(bytez): + """Calculate hash based on `bytez`. Reproduce `hash method from Facebook fastText implementation `_. Parameters ---------- - string : str - The string whose hash needs to be calculated. + bytez : bytes + The string whose hash needs to be calculated, encoded as UTF-8. Returns ------- @@ -51,8 +65,8 @@ def _ft_hash_py(string): """ old_settings = np.seterr(all='ignore') h = np.uint32(2166136261) - for c in string.encode('utf-8'): - h = h ^ np.uint32(np.int8(_byte_to_int(c))) + for b in bytez: + h = h ^ np.uint32(np.int8(_byte_to_int(b))) h = h * np.uint32(16777619) np.seterr(**old_settings) return h @@ -88,44 +102,80 @@ def _ft_hash_py_broken(string): try: from gensim.models._utils_any2vec import ( - ft_hash as _ft_hash_cy, + compute_ngrams, ft_hash_broken as _ft_hash_cy_broken, - compute_ngrams as _compute_ngrams + ft_hash_bytes as _ft_hash_cy_bytes, ) - _ft_hash = _ft_hash_cy + _ft_hash_bytes = _ft_hash_cy_bytes _ft_hash_broken = _ft_hash_cy_broken + FAST_VERSION = 0 except ImportError: + # failed... fall back to plain python FAST_VERSION = -1 - - _ft_hash = _ft_hash_py + _ft_hash_bytes = _ft_hash_py_bytes _ft_hash_broken = _ft_hash_py_broken - # failed... fall back to plain python - def _compute_ngrams(word, min_n, max_n): - """Get the list of all possible ngrams for a given word. - Parameters - ---------- - word : str - The word whose ngrams need to be computed. - min_n : int - Minimum character length of the ngrams. - max_n : int - Maximum character length of the ngrams. - - Returns - ------- - list of str - Sequence of character ngrams. +def ft_ngram_hashes(word, minn, maxn, num_buckets): + """Calculate the ngrams of the word and hash them. - """ - BOW, EOW = ('<', '>') # Used by FastText to attach to all words as prefix and suffix - extended_word = BOW + word + EOW - ngrams = [] - for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): - for i in range(0, len(extended_word) - ngram_length + 1): - ngrams.append(extended_word[i:i + ngram_length]) - return ngrams + Do this in a way that is compatible with the original Facebook implementation. + + Parameters + ---------- + word : str + The word to calculate ngram hashes for. + minn : int + Minimum ngram length + maxn : int + Maximum ngram length + num_buckets : int + The number of buckets + + Returns + ------- + A list of hashes (integers), one per each detected ngram. + + """ + ngrams = compute_ngrams(word, minn, maxn) + # + # This is a trick to avoid encoding each ngram separately, which is + # computationally expensive. It works for two reasons: + # + # 1) ngrams are guaranteed to _not_ have any spaces, because that is the + # character we use to split sentences into words. + # 2) space is an ASCII character, so it survives UTF-8 encoding unchanged. + # + encoded_ngrams = (" ".join(ngrams)).encode("utf-8").split() + hashes = [_ft_hash_bytes(en) % num_buckets for en in encoded_ngrams] + return hashes + + +def ft_ngram_hashes_broken(word, minn, maxn, num_buckets): + """Calculate the ngrams of the word and hash them. + + Do this in a way that is incompatible with the original Facebook + implementation, but compatible with older versions of Gensim. + + Parameters + ---------- + word : str + The word to calculate ngram hashes for. + minn : int + Minimum ngram length + maxn : int + Maximum ngram length + num_buckets : int + The number of buckets + + Returns + ------- + A list of hashes (integers), one per each detected ngram. + + """ + ngrams = compute_ngrams(word, minn, maxn) + hashes = [_ft_hash_broken(n) % num_buckets for n in ngrams] + return hashes def _save_word2vec_format(fname, vocab, vectors, fvocab=None, binary=False, total_vec=None): diff --git a/gensim/test/test_utils.py b/gensim/test/test_utils.py index 74ca8b941f..0be21b94ca 100644 --- a/gensim/test/test_utils.py +++ b/gensim/test/test_utils.py @@ -269,12 +269,14 @@ def hash_main(alg): import six assert six.PY3, 'this only works under Py3' + assert gensim.models.utils_any2vec.FAST_VERSION == 0, 'Cython extensions are broken' hashmap = { - 'py': gensim.models.utils_any2vec._ft_hash_py, 'py_broken': gensim.models.utils_any2vec._ft_hash_py_broken, - 'cy': gensim.models.utils_any2vec._ft_hash_py, + 'py_bytes': gensim.models.utils_any2vec._ft_hash_py_bytes, + 'cy': gensim.models.utils_any2vec._ft_hash_cy, 'cy_broken': gensim.models.utils_any2vec._ft_hash_py_broken, + 'cy_bytes': gensim.models.utils_any2vec._ft_hash_cy_bytes, } try: fun = hashmap[alg] @@ -282,7 +284,11 @@ def hash_main(alg): raise KeyError('invalid alg: %r expected one of %r' % (alg, sorted(hashmap))) for line in sys.stdin: - for word in line.rstrip().split(' '): + if 'bytes' in alg: + words = line.encode('utf-8').rstrip().split(b' ') + else: + words = line.rstrip().split(' ') + for word in words: print('u%r: %r,' % (word, fun(word))) @@ -293,7 +299,7 @@ def setUp(self): # # $ echo word1 ... wordN | python -c 'from gensim.test.test_utils import hash_main;hash_main("alg")' # noqa: E501 # - # where alg is one of py, py_broken, cy, cy_broken. + # where alg is one of py_bytes, py_broken, cy_bytes, cy_broken. # self.expected = { @@ -330,12 +336,12 @@ def setUp(self): } def test_python(self): - actual = {k: gensim.models.utils_any2vec._ft_hash_py(k) for k in self.expected} + actual = {k: gensim.models.utils_any2vec._ft_hash_py_bytes(k.encode('utf-8')) for k in self.expected} self.assertEqual(self.expected, actual) @unittest.skipIf(DISABLE_CYTHON_TESTS, 'Cython functions are not properly compiled') def test_cython(self): - actual = {k: gensim.models.utils_any2vec._ft_hash_cy(k) for k in self.expected} + actual = {k: gensim.models.utils_any2vec._ft_hash_cy_bytes(k.encode('utf-8')) for k in self.expected} self.assertEqual(self.expected, actual) def test_python_broken(self): From e458ff03bb2c9bad2603fe9b2cdf8c8d4490b0ce Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Sun, 20 Jan 2019 19:30:19 +1100 Subject: [PATCH 05/32] remove obsolete ft_hash function --- gensim/models/_utils_any2vec.c | 1881 ++++++++++++------------------ gensim/models/_utils_any2vec.pyx | 28 - 2 files changed, 758 insertions(+), 1151 deletions(-) diff --git a/gensim/models/_utils_any2vec.c b/gensim/models/_utils_any2vec.c index d78f295799..3f498f12e2 100644 --- a/gensim/models/_utils_any2vec.c +++ b/gensim/models/_utils_any2vec.c @@ -1199,75 +1199,6 @@ static CYTHON_INLINE Py_UCS4 __Pyx_PyUnicode_AsPy_UCS4(PyObject*); #endif static long __Pyx__PyObject_Ord(PyObject* c); -/* GetModuleGlobalName.proto */ -#if CYTHON_USE_DICT_VERSIONS -#define __Pyx_GetModuleGlobalName(var, name) {\ - static PY_UINT64_T __pyx_dict_version = 0;\ - static PyObject *__pyx_dict_cached_value = NULL;\ - (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_d))) ?\ - (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ - __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ -} -#define __Pyx_GetModuleGlobalNameUncached(var, name) {\ - PY_UINT64_T __pyx_dict_version;\ - PyObject *__pyx_dict_cached_value;\ - (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ -} -static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); -#else -#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) -#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) -static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); -#endif - -/* PyCFunctionFastCall.proto */ -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); -#else -#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) -#endif - -/* PyFunctionFastCall.proto */ -#if CYTHON_FAST_PYCALL -#define __Pyx_PyFunction_FastCall(func, args, nargs)\ - __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); -#else -#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) -#endif -#define __Pyx_BUILD_ASSERT_EXPR(cond)\ - (sizeof(char [1 - 2*!(cond)]) - 1) -#ifndef Py_MEMBER_SIZE -#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) -#endif - static size_t __pyx_pyframe_localsplus_offset = 0; - #include "frameobject.h" - #define __Pxy_PyFrame_Initialize_Offsets()\ - ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\ - (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) - #define __Pyx_PyFrame_GetLocalsplus(frame)\ - (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) -#endif - -/* PyObjectCall.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); -#else -#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) -#endif - -/* PyObjectCall2Args.proto */ -static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); - -/* PyObjectCallMethO.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); -#endif - -/* PyObjectCallOneArg.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); - /* ArgTypeTest.proto */ #define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ ((likely((Py_TYPE(obj) == type) | (none_allowed && (obj == Py_None)))) ? 1 :\ @@ -1307,6 +1238,13 @@ static CYTHON_INLINE int __Pyx_init_unicode_iteration( static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_count, Py_ssize_t result_ulength, Py_UCS4 max_char); +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + /* PyIntBinop.proto */ #if !CYTHON_COMPILING_IN_PYPY static PyObject* __Pyx_PyInt_AddObjC(PyObject *op1, PyObject *op2, long intval, int inplace); @@ -1387,6 +1325,44 @@ static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject /* RaiseException.proto */ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); +/* PyCFunctionFastCall.proto */ +#if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); +#else +#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) +#endif + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); +#else +#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) +#endif +#define __Pyx_BUILD_ASSERT_EXPR(cond)\ + (sizeof(char [1 - 2*!(cond)]) - 1) +#ifndef Py_MEMBER_SIZE +#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) +#endif + static size_t __pyx_pyframe_localsplus_offset = 0; + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets()\ + ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\ + (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) + #define __Pyx_PyFrame_GetLocalsplus(frame)\ + (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) +#endif + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectCallOneArg.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + /* DictGetItem.proto */ #if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key); @@ -1523,6 +1499,27 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); /* ImportFrom.proto */ static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); +/* GetModuleGlobalName.proto */ +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_d))) ?\ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} +#define __Pyx_GetModuleGlobalNameUncached(var, name) {\ + PY_UINT64_T __pyx_dict_version;\ + PyObject *__pyx_dict_cached_value;\ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); +#else +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); +#endif + /* CLineInTraceback.proto */ #ifdef CYTHON_CLINE_IN_TRACEBACK #define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) @@ -1553,10 +1550,10 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); /* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value); +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint32_t(uint32_t value); /* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint32_t(uint32_t value); +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value); /* RealImag.proto */ #if CYTHON_CCOMPLEX @@ -1729,7 +1726,6 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *, cha static PyTypeObject *__pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py = 0; static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3(PyObject *); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObject *); /*proto*/ -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject *, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObject *, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObject *, unsigned int, unsigned int, int __pyx_skip_dispatch); /*proto*/ @@ -1749,7 +1745,6 @@ static const char __pyx_k__2[] = ">"; static const char __pyx_k_np[] = "np"; static const char __pyx_k_PY2[] = "PY2"; static const char __pyx_k_six[] = "six"; -static const char __pyx_k_int8[] = "int8"; static const char __pyx_k_main[] = "__main__"; static const char __pyx_k_name[] = "__name__"; static const char __pyx_k_test[] = "__test__"; @@ -1760,7 +1755,6 @@ static const char __pyx_k_min_n[] = "min_n"; static const char __pyx_k_numpy[] = "numpy"; static const char __pyx_k_range[] = "range"; static const char __pyx_k_import[] = "__import__"; -static const char __pyx_k_uint32[] = "uint32"; static const char __pyx_k_ValueError[] = "ValueError"; static const char __pyx_k_ImportError[] = "ImportError"; static const char __pyx_k_byte_to_int[] = "_byte_to_int"; @@ -1793,7 +1787,6 @@ static PyObject *__pyx_n_s_byte_to_int; static PyObject *__pyx_n_s_cfunc_to_py; static PyObject *__pyx_n_s_cline_in_traceback; static PyObject *__pyx_n_s_import; -static PyObject *__pyx_n_s_int8; static PyObject *__pyx_n_s_main; static PyObject *__pyx_n_s_max_n; static PyObject *__pyx_n_s_min_n; @@ -1808,21 +1801,18 @@ static PyObject *__pyx_n_s_range; static PyObject *__pyx_n_s_six; static PyObject *__pyx_kp_s_stringsource; static PyObject *__pyx_n_s_test; -static PyObject *__pyx_n_s_uint32; static PyObject *__pyx_kp_u_unknown_dtype_code_in_numpy_pxd; static PyObject *__pyx_n_s_word; static PyObject *__pyx_n_s_wrap; -static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string); /* proto */ -static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_bytez); /* proto */ -static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4ft_hash_broken(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string); /* proto */ -static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_bytez); /* proto */ +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string); /* proto */ +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info); /* proto */ static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(PyObject *__pyx_self, PyObject *__pyx_v_b); /* proto */ static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_int_0; static PyObject *__pyx_int_1; -static PyObject *__pyx_int_16777619; static PyObject *__pyx_tuple__3; static PyObject *__pyx_tuple__4; static PyObject *__pyx_tuple__5; @@ -1926,414 +1916,92 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObj /* "gensim/models/_utils_any2vec.pyx":25 * * - * cpdef ft_hash(unicode string): # <<<<<<<<<<<<<< - * """Calculate hash based on `string`. + * cpdef ft_hash_bytes(bytes bytez): # <<<<<<<<<<<<<< + * """Calculate hash based on `bytez`. * Reproduce `hash method from Facebook fastText implementation */ -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash(PyObject *__pyx_self, PyObject *__pyx_v_string); /*proto*/ -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(PyObject *__pyx_v_string, CYTHON_UNUSED int __pyx_skip_dispatch) { - unsigned int __pyx_v_h; - PyObject *__pyx_v_c = NULL; +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash_bytes(PyObject *__pyx_self, PyObject *__pyx_v_bytez); /*proto*/ +static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject *__pyx_v_bytez, CYTHON_UNUSED int __pyx_skip_dispatch) { + uint32_t __pyx_v_h; + char __pyx_v_b; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *(*__pyx_t_3)(PyObject *); - PyObject *__pyx_t_4 = NULL; - PyObject *__pyx_t_5 = NULL; + char *__pyx_t_2; + char *__pyx_t_3; + char *__pyx_t_4; + char *__pyx_t_5; PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - PyObject *__pyx_t_8 = NULL; - PyObject *__pyx_t_9 = NULL; - PyObject *__pyx_t_10 = NULL; - PyObject *__pyx_t_11 = NULL; - PyObject *__pyx_t_12 = NULL; - unsigned int __pyx_t_13; - __Pyx_RefNannySetupContext("ft_hash", 0); + __Pyx_RefNannySetupContext("ft_hash_bytes", 0); - /* "gensim/models/_utils_any2vec.pyx":46 + /* "gensim/models/_utils_any2vec.pyx":41 * * """ - * cdef unsigned int h = 2166136261 # <<<<<<<<<<<<<< - * for c in string.encode("utf-8"): - * h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) + * cdef uint32_t h = 2166136261 # <<<<<<<<<<<<<< + * cdef char b + * */ __pyx_v_h = 0x811C9DC5; - /* "gensim/models/_utils_any2vec.pyx":47 - * """ - * cdef unsigned int h = 2166136261 - * for c in string.encode("utf-8"): # <<<<<<<<<<<<<< - * h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) - * h = np.uint32(h * np.uint32(16777619)) + /* "gensim/models/_utils_any2vec.pyx":44 + * cdef char b + * + * for b in bytez: # <<<<<<<<<<<<<< + * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss + * h = h * 16777619 */ - if (unlikely(__pyx_v_string == Py_None)) { - PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%.30s'", "encode"); - __PYX_ERR(0, 47, __pyx_L1_error) + if (unlikely(__pyx_v_bytez == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); + __PYX_ERR(0, 44, __pyx_L1_error) } - __pyx_t_1 = PyUnicode_AsUTF8String(__pyx_v_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 47, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 47, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = Py_TYPE(__pyx_t_2)->tp_iternext; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 47, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - for (;;) { - { - __pyx_t_1 = __pyx_t_3(__pyx_t_2); - if (unlikely(!__pyx_t_1)) { - PyObject* exc_type = PyErr_Occurred(); - if (exc_type) { - if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(0, 47, __pyx_L1_error) - } - break; - } - __Pyx_GOTREF(__pyx_t_1); - } - __Pyx_XDECREF_SET(__pyx_v_c, __pyx_t_1); - __pyx_t_1 = 0; + __Pyx_INCREF(__pyx_v_bytez); + __pyx_t_1 = __pyx_v_bytez; + __pyx_t_3 = PyBytes_AS_STRING(__pyx_t_1); + __pyx_t_4 = (__pyx_t_3 + PyBytes_GET_SIZE(__pyx_t_1)); + for (__pyx_t_5 = __pyx_t_3; __pyx_t_5 < __pyx_t_4; __pyx_t_5++) { + __pyx_t_2 = __pyx_t_5; + __pyx_v_b = (__pyx_t_2[0]); - /* "gensim/models/_utils_any2vec.pyx":48 - * cdef unsigned int h = 2166136261 - * for c in string.encode("utf-8"): - * h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) # <<<<<<<<<<<<<< - * h = np.uint32(h * np.uint32(16777619)) + /* "gensim/models/_utils_any2vec.pyx":45 + * + * for b in bytez: + * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss # <<<<<<<<<<<<<< + * h = h * 16777619 * return h */ - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_np); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 48, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_uint32); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 48, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 48, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_n_s_np); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 48, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_uint32); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 48, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_n_s_np); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 48, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_9); - __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_n_s_int8); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 48, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_10); - __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - __Pyx_GetModuleGlobalName(__pyx_t_11, __pyx_n_s_byte_to_int); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 48, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_11); - __pyx_t_12 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_11))) { - __pyx_t_12 = PyMethod_GET_SELF(__pyx_t_11); - if (likely(__pyx_t_12)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); - __Pyx_INCREF(__pyx_t_12); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_11, function); - } - } - __pyx_t_9 = (__pyx_t_12) ? __Pyx_PyObject_Call2Args(__pyx_t_11, __pyx_t_12, __pyx_v_c) : __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_v_c); - __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0; - if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 48, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_9); - __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; - __pyx_t_11 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_10))) { - __pyx_t_11 = PyMethod_GET_SELF(__pyx_t_10); - if (likely(__pyx_t_11)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_10); - __Pyx_INCREF(__pyx_t_11); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_10, function); - } - } - __pyx_t_7 = (__pyx_t_11) ? __Pyx_PyObject_Call2Args(__pyx_t_10, __pyx_t_11, __pyx_t_9) : __Pyx_PyObject_CallOneArg(__pyx_t_10, __pyx_t_9); - __Pyx_XDECREF(__pyx_t_11); __pyx_t_11 = 0; - __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 48, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - __pyx_t_10 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_8))) { - __pyx_t_10 = PyMethod_GET_SELF(__pyx_t_8); - if (likely(__pyx_t_10)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_8); - __Pyx_INCREF(__pyx_t_10); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_8, function); - } - } - __pyx_t_6 = (__pyx_t_10) ? __Pyx_PyObject_Call2Args(__pyx_t_8, __pyx_t_10, __pyx_t_7) : __Pyx_PyObject_CallOneArg(__pyx_t_8, __pyx_t_7); - __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 48, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_8 = PyNumber_Xor(__pyx_t_4, __pyx_t_6); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 48, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_5))) { - __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_5); - if (likely(__pyx_t_6)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_5); - __Pyx_INCREF(__pyx_t_6); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_5, function); - } - } - __pyx_t_1 = (__pyx_t_6) ? __Pyx_PyObject_Call2Args(__pyx_t_5, __pyx_t_6, __pyx_t_8) : __Pyx_PyObject_CallOneArg(__pyx_t_5, __pyx_t_8); - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 48, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_13 = __Pyx_PyInt_As_unsigned_int(__pyx_t_1); if (unlikely((__pyx_t_13 == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 48, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_v_h = __pyx_t_13; + __pyx_v_h = (__pyx_v_h ^ ((uint32_t)((int8_t)__pyx_v_b))); - /* "gensim/models/_utils_any2vec.pyx":49 - * for c in string.encode("utf-8"): - * h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) - * h = np.uint32(h * np.uint32(16777619)) # <<<<<<<<<<<<<< + /* "gensim/models/_utils_any2vec.pyx":46 + * for b in bytez: + * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss + * h = h * 16777619 # <<<<<<<<<<<<<< * return h * */ - __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_n_s_np); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 49, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_uint32); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 49, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 49, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_np); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 49, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_uint32); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 49, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { - __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_7); - if (likely(__pyx_t_4)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_7); - __Pyx_INCREF(__pyx_t_4); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_7, function); - } - } - __pyx_t_6 = (__pyx_t_4) ? __Pyx_PyObject_Call2Args(__pyx_t_7, __pyx_t_4, __pyx_int_16777619) : __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_int_16777619); - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 49, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = PyNumber_Multiply(__pyx_t_5, __pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 49, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_8))) { - __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_8); - if (likely(__pyx_t_6)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_8); - __Pyx_INCREF(__pyx_t_6); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_8, function); - } - } - __pyx_t_1 = (__pyx_t_6) ? __Pyx_PyObject_Call2Args(__pyx_t_8, __pyx_t_6, __pyx_t_7) : __Pyx_PyObject_CallOneArg(__pyx_t_8, __pyx_t_7); - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 49, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_13 = __Pyx_PyInt_As_unsigned_int(__pyx_t_1); if (unlikely((__pyx_t_13 == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 49, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_v_h = __pyx_t_13; - - /* "gensim/models/_utils_any2vec.pyx":47 - * """ - * cdef unsigned int h = 2166136261 - * for c in string.encode("utf-8"): # <<<<<<<<<<<<<< - * h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) - * h = np.uint32(h * np.uint32(16777619)) - */ + __pyx_v_h = (__pyx_v_h * 0x1000193); } - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":50 - * h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) - * h = np.uint32(h * np.uint32(16777619)) + /* "gensim/models/_utils_any2vec.pyx":47 + * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss + * h = h * 16777619 * return h # <<<<<<<<<<<<<< * * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 50, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; + __pyx_t_6 = __Pyx_PyInt_From_uint32_t(__pyx_v_h); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 47, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; goto __pyx_L0; /* "gensim/models/_utils_any2vec.pyx":25 * * - * cpdef ft_hash(unicode string): # <<<<<<<<<<<<<< - * """Calculate hash based on `string`. - * Reproduce `hash method from Facebook fastText implementation - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_XDECREF(__pyx_t_5); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_XDECREF(__pyx_t_8); - __Pyx_XDECREF(__pyx_t_9); - __Pyx_XDECREF(__pyx_t_10); - __Pyx_XDECREF(__pyx_t_11); - __Pyx_XDECREF(__pyx_t_12); - __Pyx_AddTraceback("gensim.models._utils_any2vec.ft_hash", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_c); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* Python wrapper */ -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash(PyObject *__pyx_self, PyObject *__pyx_v_string); /*proto*/ -static char __pyx_doc_6gensim_6models_14_utils_any2vec_ft_hash[] = "ft_hash(unicode string)\nCalculate hash based on `string`.\n Reproduce `hash method from Facebook fastText implementation\n `_.\n\n This function is expensive because it encodes each input string as UTF-8,\n and this can accumulate to a high computational cost if there are many\n strings. If possible, convert your strings to bytes and use the faster\n :py:func:`ft_hash_bytes` function instead.\n\n Parameters\n ----------\n string : unicode\n The string whose hash needs to be calculated.\n\n Returns\n -------\n unsigned int\n The hash of the string.\n\n "; -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash(PyObject *__pyx_self, PyObject *__pyx_v_string) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("ft_hash (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_string), (&PyUnicode_Type), 1, "string", 1))) __PYX_ERR(0, 25, __pyx_L1_error) - __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash(__pyx_self, ((PyObject*)__pyx_v_string)); - - /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = NULL; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("ft_hash", 0); - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash(__pyx_v_string, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 25, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("gensim.models._utils_any2vec.ft_hash", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "gensim/models/_utils_any2vec.pyx":53 - * - * - * cpdef ft_hash_bytes(bytes bytez): # <<<<<<<<<<<<<< - * """Calculate hash based on `bytez`. - * Reproduce `hash method from Facebook fastText implementation - */ - -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_bytes(PyObject *__pyx_self, PyObject *__pyx_v_bytez); /*proto*/ -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject *__pyx_v_bytez, CYTHON_UNUSED int __pyx_skip_dispatch) { - uint32_t __pyx_v_h; - char __pyx_v_b; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - char *__pyx_t_2; - char *__pyx_t_3; - char *__pyx_t_4; - char *__pyx_t_5; - PyObject *__pyx_t_6 = NULL; - __Pyx_RefNannySetupContext("ft_hash_bytes", 0); - - /* "gensim/models/_utils_any2vec.pyx":69 - * - * """ - * cdef uint32_t h = 2166136261 # <<<<<<<<<<<<<< - * cdef char b - * - */ - __pyx_v_h = 0x811C9DC5; - - /* "gensim/models/_utils_any2vec.pyx":72 - * cdef char b - * - * for b in bytez: # <<<<<<<<<<<<<< - * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss - * h = h * 16777619 - */ - if (unlikely(__pyx_v_bytez == Py_None)) { - PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); - __PYX_ERR(0, 72, __pyx_L1_error) - } - __Pyx_INCREF(__pyx_v_bytez); - __pyx_t_1 = __pyx_v_bytez; - __pyx_t_3 = PyBytes_AS_STRING(__pyx_t_1); - __pyx_t_4 = (__pyx_t_3 + PyBytes_GET_SIZE(__pyx_t_1)); - for (__pyx_t_5 = __pyx_t_3; __pyx_t_5 < __pyx_t_4; __pyx_t_5++) { - __pyx_t_2 = __pyx_t_5; - __pyx_v_b = (__pyx_t_2[0]); - - /* "gensim/models/_utils_any2vec.pyx":73 - * - * for b in bytez: - * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss # <<<<<<<<<<<<<< - * h = h * 16777619 - * return h - */ - __pyx_v_h = (__pyx_v_h ^ ((uint32_t)((int8_t)__pyx_v_b))); - - /* "gensim/models/_utils_any2vec.pyx":74 - * for b in bytez: - * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss - * h = h * 16777619 # <<<<<<<<<<<<<< - * return h - * - */ - __pyx_v_h = (__pyx_v_h * 0x1000193); - } - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "gensim/models/_utils_any2vec.pyx":75 - * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss - * h = h * 16777619 - * return h # <<<<<<<<<<<<<< - * - * - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_6 = __Pyx_PyInt_From_uint32_t(__pyx_v_h); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 75, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __pyx_r = __pyx_t_6; - __pyx_t_6 = 0; - goto __pyx_L0; - - /* "gensim/models/_utils_any2vec.pyx":53 - * - * - * cpdef ft_hash_bytes(bytes bytez): # <<<<<<<<<<<<<< - * """Calculate hash based on `bytez`. + * cpdef ft_hash_bytes(bytes bytez): # <<<<<<<<<<<<<< + * """Calculate hash based on `bytez`. * Reproduce `hash method from Facebook fastText implementation */ @@ -2350,14 +2018,14 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject } /* Python wrapper */ -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_bytes(PyObject *__pyx_self, PyObject *__pyx_v_bytez); /*proto*/ -static char __pyx_doc_6gensim_6models_14_utils_any2vec_2ft_hash_bytes[] = "ft_hash_bytes(bytes bytez)\nCalculate hash based on `bytez`.\n Reproduce `hash method from Facebook fastText implementation\n `_.\n\n Parameters\n ----------\n bytez : bytes\n The string whose hash needs to be calculated, encoded as UTF-8.\n\n Returns\n -------\n unsigned int\n The hash of the string.\n\n "; -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_bytes(PyObject *__pyx_self, PyObject *__pyx_v_bytez) { +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash_bytes(PyObject *__pyx_self, PyObject *__pyx_v_bytez); /*proto*/ +static char __pyx_doc_6gensim_6models_14_utils_any2vec_ft_hash_bytes[] = "ft_hash_bytes(bytes bytez)\nCalculate hash based on `bytez`.\n Reproduce `hash method from Facebook fastText implementation\n `_.\n\n Parameters\n ----------\n bytez : bytes\n The string whose hash needs to be calculated, encoded as UTF-8.\n\n Returns\n -------\n unsigned int\n The hash of the string.\n\n "; +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash_bytes(PyObject *__pyx_self, PyObject *__pyx_v_bytez) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("ft_hash_bytes (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_bytez), (&PyBytes_Type), 1, "bytez", 1))) __PYX_ERR(0, 53, __pyx_L1_error) - __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_bytes(__pyx_self, ((PyObject*)__pyx_v_bytez)); + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_bytez), (&PyBytes_Type), 1, "bytez", 1))) __PYX_ERR(0, 25, __pyx_L1_error) + __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(__pyx_self, ((PyObject*)__pyx_v_bytez)); /* function exit code */ goto __pyx_L0; @@ -2368,13 +2036,13 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_bytes(PyObje return __pyx_r; } -static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_bytez) { +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_bytez) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("ft_hash_bytes", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(__pyx_v_bytez, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 53, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(__pyx_v_bytez, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 25, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -2391,7 +2059,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_bytes(CYTHON return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":78 +/* "gensim/models/_utils_any2vec.pyx":50 * * * cpdef ft_hash_broken(unicode string): # <<<<<<<<<<<<<< @@ -2399,7 +2067,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_bytes(CYTHON * */ -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5ft_hash_broken(PyObject *__pyx_self, PyObject *__pyx_v_string); /*proto*/ +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken(PyObject *__pyx_self, PyObject *__pyx_v_string); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObject *__pyx_v_string, CYTHON_UNUSED int __pyx_skip_dispatch) { unsigned int __pyx_v_h; PyObject *__pyx_v_c = NULL; @@ -2416,7 +2084,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec long __pyx_t_9; __Pyx_RefNannySetupContext("ft_hash_broken", 0); - /* "gensim/models/_utils_any2vec.pyx":95 + /* "gensim/models/_utils_any2vec.pyx":67 * * """ * cdef unsigned int h = 2166136261 # <<<<<<<<<<<<<< @@ -2425,7 +2093,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec */ __pyx_v_h = 0x811C9DC5; - /* "gensim/models/_utils_any2vec.pyx":96 + /* "gensim/models/_utils_any2vec.pyx":68 * """ * cdef unsigned int h = 2166136261 * for c in string: # <<<<<<<<<<<<<< @@ -2434,29 +2102,29 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec */ if (unlikely(__pyx_v_string == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); - __PYX_ERR(0, 96, __pyx_L1_error) + __PYX_ERR(0, 68, __pyx_L1_error) } __Pyx_INCREF(__pyx_v_string); __pyx_t_1 = __pyx_v_string; - __pyx_t_6 = __Pyx_init_unicode_iteration(__pyx_t_1, (&__pyx_t_3), (&__pyx_t_4), (&__pyx_t_5)); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 96, __pyx_L1_error) + __pyx_t_6 = __Pyx_init_unicode_iteration(__pyx_t_1, (&__pyx_t_3), (&__pyx_t_4), (&__pyx_t_5)); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 68, __pyx_L1_error) for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_3; __pyx_t_7++) { __pyx_t_2 = __pyx_t_7; - __pyx_t_8 = PyUnicode_FromOrdinal(__Pyx_PyUnicode_READ(__pyx_t_5, __pyx_t_4, __pyx_t_2)); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 96, __pyx_L1_error) + __pyx_t_8 = PyUnicode_FromOrdinal(__Pyx_PyUnicode_READ(__pyx_t_5, __pyx_t_4, __pyx_t_2)); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 68, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_XDECREF_SET(__pyx_v_c, __pyx_t_8); __pyx_t_8 = 0; - /* "gensim/models/_utils_any2vec.pyx":97 + /* "gensim/models/_utils_any2vec.pyx":69 * cdef unsigned int h = 2166136261 * for c in string: * h ^= ord(c) # <<<<<<<<<<<<<< * h *= 16777619 * return h */ - __pyx_t_9 = __Pyx_PyObject_Ord(__pyx_v_c); if (unlikely(__pyx_t_9 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyObject_Ord(__pyx_v_c); if (unlikely(__pyx_t_9 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 69, __pyx_L1_error) __pyx_v_h = (__pyx_v_h ^ __pyx_t_9); - /* "gensim/models/_utils_any2vec.pyx":98 + /* "gensim/models/_utils_any2vec.pyx":70 * for c in string: * h ^= ord(c) * h *= 16777619 # <<<<<<<<<<<<<< @@ -2467,7 +2135,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":99 + /* "gensim/models/_utils_any2vec.pyx":71 * h ^= ord(c) * h *= 16777619 * return h # <<<<<<<<<<<<<< @@ -2475,13 +2143,13 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_8 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 99, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 71, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __pyx_r = __pyx_t_8; __pyx_t_8 = 0; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":78 + /* "gensim/models/_utils_any2vec.pyx":50 * * * cpdef ft_hash_broken(unicode string): # <<<<<<<<<<<<<< @@ -2503,14 +2171,14 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec } /* Python wrapper */ -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5ft_hash_broken(PyObject *__pyx_self, PyObject *__pyx_v_string); /*proto*/ -static char __pyx_doc_6gensim_6models_14_utils_any2vec_4ft_hash_broken[] = "ft_hash_broken(unicode string)\nCalculate hash based on `string`.\n\n This implementation is broken, see https://github.com/RaRe-Technologies/gensim/issues/2059.\n It is here only for maintaining backwards compatibility with older models.\n\n Parameters\n ----------\n string : unicode\n The string whose hash needs to be calculated.\n\n Returns\n -------\n unsigned int\n The hash of the string.\n\n "; -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5ft_hash_broken(PyObject *__pyx_self, PyObject *__pyx_v_string) { +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken(PyObject *__pyx_self, PyObject *__pyx_v_string); /*proto*/ +static char __pyx_doc_6gensim_6models_14_utils_any2vec_2ft_hash_broken[] = "ft_hash_broken(unicode string)\nCalculate hash based on `string`.\n\n This implementation is broken, see https://github.com/RaRe-Technologies/gensim/issues/2059.\n It is here only for maintaining backwards compatibility with older models.\n\n Parameters\n ----------\n string : unicode\n The string whose hash needs to be calculated.\n\n Returns\n -------\n unsigned int\n The hash of the string.\n\n "; +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken(PyObject *__pyx_self, PyObject *__pyx_v_string) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("ft_hash_broken (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_string), (&PyUnicode_Type), 1, "string", 1))) __PYX_ERR(0, 78, __pyx_L1_error) - __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_4ft_hash_broken(__pyx_self, ((PyObject*)__pyx_v_string)); + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_string), (&PyUnicode_Type), 1, "string", 1))) __PYX_ERR(0, 50, __pyx_L1_error) + __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(__pyx_self, ((PyObject*)__pyx_v_string)); /* function exit code */ goto __pyx_L0; @@ -2521,13 +2189,13 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5ft_hash_broken(PyObj return __pyx_r; } -static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4ft_hash_broken(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string) { +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("ft_hash_broken", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(__pyx_v_string, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 78, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(__pyx_v_string, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 50, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -2544,7 +2212,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4ft_hash_broken(CYTHO return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":102 +/* "gensim/models/_utils_any2vec.pyx":74 * * * cpdef compute_ngrams(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -2552,7 +2220,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4ft_hash_broken(CYTHO * */ -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n, CYTHON_UNUSED int __pyx_skip_dispatch) { PyObject *__pyx_v_extended_word = 0; PyObject *__pyx_v_ngrams = NULL; @@ -2576,14 +2244,14 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec int __pyx_t_14; __Pyx_RefNannySetupContext("compute_ngrams", 0); - /* "gensim/models/_utils_any2vec.pyx":120 + /* "gensim/models/_utils_any2vec.pyx":92 * * """ * cdef unicode extended_word = f'<{word}>' # <<<<<<<<<<<<<< * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): */ - __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 120, __pyx_L1_error) + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = 0; __pyx_t_3 = 127; @@ -2591,7 +2259,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __pyx_t_2 += 1; __Pyx_GIVEREF(__pyx_kp_u_); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_kp_u_); - __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_word, __pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 120, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_word, __pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = (__Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) > __pyx_t_3) ? __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) : __pyx_t_3; __pyx_t_2 += __Pyx_PyUnicode_GET_LENGTH(__pyx_t_4); @@ -2602,43 +2270,43 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __pyx_t_2 += 1; __Pyx_GIVEREF(__pyx_kp_u__2); PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_kp_u__2); - __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_1, 3, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 120, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_1, 3, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_extended_word = ((PyObject*)__pyx_t_4); __pyx_t_4 = 0; - /* "gensim/models/_utils_any2vec.pyx":121 + /* "gensim/models/_utils_any2vec.pyx":93 * """ * cdef unicode extended_word = f'<{word}>' * ngrams = [] # <<<<<<<<<<<<<< * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): */ - __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 121, __pyx_L1_error) + __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 93, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_v_ngrams = ((PyObject*)__pyx_t_4); __pyx_t_4 = 0; - /* "gensim/models/_utils_any2vec.pyx":122 + /* "gensim/models/_utils_any2vec.pyx":94 * cdef unicode extended_word = f'<{word}>' * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): # <<<<<<<<<<<<<< * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) */ - __pyx_t_4 = __Pyx_PyInt_From_unsigned_int(__pyx_v_min_n); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 122, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyInt_From_unsigned_int(__pyx_v_min_n); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 94, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = __pyx_v_max_n; - __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 122, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 94, __pyx_L1_error) if (((__pyx_t_5 < __pyx_t_2) != 0)) { __pyx_t_6 = __pyx_t_5; } else { __pyx_t_6 = __pyx_t_2; } - __pyx_t_1 = PyInt_FromSsize_t((__pyx_t_6 + 1)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 122, __pyx_L1_error) + __pyx_t_1 = PyInt_FromSsize_t((__pyx_t_6 + 1)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 94, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 122, __pyx_L1_error) + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 94, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_4); @@ -2646,16 +2314,16 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_1); __pyx_t_4 = 0; __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 122, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 94, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (likely(PyList_CheckExact(__pyx_t_1)) || PyTuple_CheckExact(__pyx_t_1)) { __pyx_t_7 = __pyx_t_1; __Pyx_INCREF(__pyx_t_7); __pyx_t_6 = 0; __pyx_t_8 = NULL; } else { - __pyx_t_6 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 122, __pyx_L1_error) + __pyx_t_6 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 94, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_8 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 122, __pyx_L1_error) + __pyx_t_8 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 94, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; for (;;) { @@ -2663,17 +2331,17 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec if (likely(PyList_CheckExact(__pyx_t_7))) { if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_7)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 122, __pyx_L1_error) + __pyx_t_1 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 94, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 122, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 94, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } else { if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_7)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 122, __pyx_L1_error) + __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 94, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 122, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 94, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } @@ -2683,7 +2351,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(0, 122, __pyx_L1_error) + else __PYX_ERR(0, 94, __pyx_L1_error) } break; } @@ -2692,23 +2360,23 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __Pyx_XDECREF_SET(__pyx_v_ngram_length, __pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":123 + /* "gensim/models/_utils_any2vec.pyx":95 * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): # <<<<<<<<<<<<<< * ngrams.append(extended_word[i:i + ngram_length]) * return ngrams */ - __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 123, __pyx_L1_error) - __pyx_t_1 = PyInt_FromSsize_t(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 123, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 95, __pyx_L1_error) + __pyx_t_1 = PyInt_FromSsize_t(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 95, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_4 = PyNumber_Subtract(__pyx_t_1, __pyx_v_ngram_length); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 123, __pyx_L1_error) + __pyx_t_4 = PyNumber_Subtract(__pyx_t_1, __pyx_v_ngram_length); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 95, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyInt_AddObjC(__pyx_t_4, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 123, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_AddObjC(__pyx_t_4, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 95, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 123, __pyx_L1_error) + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 95, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_INCREF(__pyx_int_0); __Pyx_GIVEREF(__pyx_int_0); @@ -2716,16 +2384,16 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 123, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 95, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (likely(PyList_CheckExact(__pyx_t_1)) || PyTuple_CheckExact(__pyx_t_1)) { __pyx_t_4 = __pyx_t_1; __Pyx_INCREF(__pyx_t_4); __pyx_t_2 = 0; __pyx_t_9 = NULL; } else { - __pyx_t_2 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 123, __pyx_L1_error) + __pyx_t_2 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 95, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_9 = Py_TYPE(__pyx_t_4)->tp_iternext; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 123, __pyx_L1_error) + __pyx_t_9 = Py_TYPE(__pyx_t_4)->tp_iternext; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 95, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; for (;;) { @@ -2733,17 +2401,17 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec if (likely(PyList_CheckExact(__pyx_t_4))) { if (__pyx_t_2 >= PyList_GET_SIZE(__pyx_t_4)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 123, __pyx_L1_error) + __pyx_t_1 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 95, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 123, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 95, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } else { if (__pyx_t_2 >= PyTuple_GET_SIZE(__pyx_t_4)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 123, __pyx_L1_error) + __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 95, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 123, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 95, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } @@ -2753,7 +2421,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(0, 123, __pyx_L1_error) + else __PYX_ERR(0, 95, __pyx_L1_error) } break; } @@ -2762,7 +2430,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __Pyx_XDECREF_SET(__pyx_v_i, __pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":124 + /* "gensim/models/_utils_any2vec.pyx":96 * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) # <<<<<<<<<<<<<< @@ -2774,26 +2442,26 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec if (__pyx_t_11) { __pyx_t_10 = 0; } else { - __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 124, __pyx_L1_error) + __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 96, __pyx_L1_error) __pyx_t_10 = __pyx_t_12; } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyNumber_Add(__pyx_v_i, __pyx_v_ngram_length); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 124, __pyx_L1_error) + __pyx_t_1 = PyNumber_Add(__pyx_v_i, __pyx_v_ngram_length); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 96, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_11 = (__pyx_t_1 == Py_None); if (__pyx_t_11) { __pyx_t_12 = PY_SSIZE_T_MAX; } else { - __pyx_t_13 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_13 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 124, __pyx_L1_error) + __pyx_t_13 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_13 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 96, __pyx_L1_error) __pyx_t_12 = __pyx_t_13; } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyUnicode_Substring(__pyx_v_extended_word, __pyx_t_10, __pyx_t_12); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 124, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyUnicode_Substring(__pyx_v_extended_word, __pyx_t_10, __pyx_t_12); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 96, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_1); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 124, __pyx_L1_error) + __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_1); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 96, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":123 + /* "gensim/models/_utils_any2vec.pyx":95 * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): # <<<<<<<<<<<<<< @@ -2803,7 +2471,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec } __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "gensim/models/_utils_any2vec.pyx":122 + /* "gensim/models/_utils_any2vec.pyx":94 * cdef unicode extended_word = f'<{word}>' * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): # <<<<<<<<<<<<<< @@ -2813,7 +2481,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - /* "gensim/models/_utils_any2vec.pyx":125 + /* "gensim/models/_utils_any2vec.pyx":97 * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) * return ngrams # <<<<<<<<<<<<<< @@ -2823,7 +2491,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __pyx_r = __pyx_v_ngrams; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":102 + /* "gensim/models/_utils_any2vec.pyx":74 * * * cpdef compute_ngrams(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -2849,9 +2517,9 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec } /* Python wrapper */ -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_6gensim_6models_14_utils_any2vec_6compute_ngrams[] = "compute_ngrams(word, unsigned int min_n, unsigned int max_n)\nGet the list of all possible ngrams for a given word.\n\n Parameters\n ----------\n word : str\n The word whose ngrams need to be computed.\n min_n : unsigned int\n Minimum character length of the ngrams.\n max_n : unsigned int\n Maximum character length of the ngrams.\n\n Returns\n -------\n list of str\n Sequence of character ngrams.\n\n "; -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_6gensim_6models_14_utils_any2vec_4compute_ngrams[] = "compute_ngrams(word, unsigned int min_n, unsigned int max_n)\nGet the list of all possible ngrams for a given word.\n\n Parameters\n ----------\n word : str\n The word whose ngrams need to be computed.\n min_n : unsigned int\n Minimum character length of the ngrams.\n max_n : unsigned int\n Maximum character length of the ngrams.\n\n Returns\n -------\n list of str\n Sequence of character ngrams.\n\n "; +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_word = 0; unsigned int __pyx_v_min_n; unsigned int __pyx_v_max_n; @@ -2883,17 +2551,17 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams(PyObj case 1: if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_min_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 1); __PYX_ERR(0, 102, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 1); __PYX_ERR(0, 74, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 2); __PYX_ERR(0, 102, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 2); __PYX_ERR(0, 74, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams") < 0)) __PYX_ERR(0, 102, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams") < 0)) __PYX_ERR(0, 74, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { goto __pyx_L5_argtuple_error; @@ -2903,31 +2571,31 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams(PyObj values[2] = PyTuple_GET_ITEM(__pyx_args, 2); } __pyx_v_word = values[0]; - __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 102, __pyx_L3_error) - __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 102, __pyx_L3_error) + __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 74, __pyx_L3_error) + __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 74, __pyx_L3_error) } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 102, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 74, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams(__pyx_self, __pyx_v_word, __pyx_v_min_n, __pyx_v_max_n); + __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(__pyx_self, __pyx_v_word, __pyx_v_min_n, __pyx_v_max_n); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n) { +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("compute_ngrams", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 102, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 74, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -5581,10 +5249,9 @@ static PyTypeObject __pyx_scope_struct____Pyx_CFunc_object____object___to_py = { }; static PyMethodDef __pyx_methods[] = { - {"ft_hash", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_ft_hash}, - {"ft_hash_bytes", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_bytes, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_2ft_hash_bytes}, - {"ft_hash_broken", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_5ft_hash_broken, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_4ft_hash_broken}, - {"compute_ngrams", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams, METH_VARARGS|METH_KEYWORDS, __pyx_doc_6gensim_6models_14_utils_any2vec_6compute_ngrams}, + {"ft_hash_bytes", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash_bytes, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_ft_hash_bytes}, + {"ft_hash_broken", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_2ft_hash_broken}, + {"compute_ngrams", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams, METH_VARARGS|METH_KEYWORDS, __pyx_doc_6gensim_6models_14_utils_any2vec_4compute_ngrams}, {0, 0, 0, 0} }; @@ -5645,7 +5312,6 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_cfunc_to_py, __pyx_k_cfunc_to_py, sizeof(__pyx_k_cfunc_to_py), 0, 0, 1, 1}, {&__pyx_n_s_cline_in_traceback, __pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 0, 1, 1}, {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, - {&__pyx_n_s_int8, __pyx_k_int8, sizeof(__pyx_k_int8), 0, 0, 1, 1}, {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, {&__pyx_n_s_max_n, __pyx_k_max_n, sizeof(__pyx_k_max_n), 0, 0, 1, 1}, {&__pyx_n_s_min_n, __pyx_k_min_n, sizeof(__pyx_k_min_n), 0, 0, 1, 1}, @@ -5660,14 +5326,13 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_six, __pyx_k_six, sizeof(__pyx_k_six), 0, 0, 1, 1}, {&__pyx_kp_s_stringsource, __pyx_k_stringsource, sizeof(__pyx_k_stringsource), 0, 0, 1, 0}, {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, - {&__pyx_n_s_uint32, __pyx_k_uint32, sizeof(__pyx_k_uint32), 0, 0, 1, 1}, {&__pyx_kp_u_unknown_dtype_code_in_numpy_pxd, __pyx_k_unknown_dtype_code_in_numpy_pxd, sizeof(__pyx_k_unknown_dtype_code_in_numpy_pxd), 0, 1, 0, 0}, {&__pyx_n_s_word, __pyx_k_word, sizeof(__pyx_k_word), 0, 0, 1, 1}, {&__pyx_n_s_wrap, __pyx_k_wrap, sizeof(__pyx_k_wrap), 0, 0, 1, 1}, {0, 0, 0, 0, 0, 0, 0} }; static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(0, 122, __pyx_L1_error) + __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(0, 94, __pyx_L1_error) __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 272, __pyx_L1_error) __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_n_s_RuntimeError); if (!__pyx_builtin_RuntimeError) __PYX_ERR(1, 856, __pyx_L1_error) __pyx_builtin_ImportError = __Pyx_GetBuiltinName(__pyx_n_s_ImportError); if (!__pyx_builtin_ImportError) __PYX_ERR(1, 1038, __pyx_L1_error) @@ -5779,7 +5444,6 @@ static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void) { if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(0, 1, __pyx_L1_error); __pyx_int_0 = PyInt_FromLong(0); if (unlikely(!__pyx_int_0)) __PYX_ERR(0, 1, __pyx_L1_error) __pyx_int_1 = PyInt_FromLong(1); if (unlikely(!__pyx_int_1)) __PYX_ERR(0, 1, __pyx_L1_error) - __pyx_int_16777619 = PyInt_FromLong(16777619L); if (unlikely(!__pyx_int_16777619)) __PYX_ERR(0, 1, __pyx_L1_error) return 0; __pyx_L1_error:; return -1; @@ -6294,183 +5958,105 @@ static long __Pyx__PyObject_Ord(PyObject* c) { return (long)(Py_UCS4)-1; } -/* GetModuleGlobalName */ -#if CYTHON_USE_DICT_VERSIONS -static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) -#else -static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) -#endif +/* ArgTypeTest */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) { - PyObject *result; -#if !CYTHON_AVOID_BORROWED_REFS -#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 - result = _PyDict_GetItem_KnownHash(__pyx_d, name, ((PyASCIIObject *) name)->hash); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } else if (unlikely(PyErr_Occurred())) { - return NULL; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; } -#else - result = PyDict_GetItem(__pyx_d, name); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); + else if (exact) { + #if PY_MAJOR_VERSION == 2 + if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; + #endif + } + else { + if (likely(__Pyx_TypeCheck(obj, type))) return 1; } + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", + name, type->tp_name, Py_TYPE(obj)->tp_name); + return 0; +} + +/* unicode_iter */ +static CYTHON_INLINE int __Pyx_init_unicode_iteration( + PyObject* ustring, Py_ssize_t *length, void** data, int *kind) { +#if CYTHON_PEP393_ENABLED + if (unlikely(__Pyx_PyUnicode_READY(ustring) < 0)) return -1; + *kind = PyUnicode_KIND(ustring); + *length = PyUnicode_GET_LENGTH(ustring); + *data = PyUnicode_DATA(ustring); +#else + *kind = 0; + *length = PyUnicode_GET_SIZE(ustring); + *data = (void*)PyUnicode_AS_UNICODE(ustring); #endif + return 0; +} + +/* JoinPyUnicode */ +static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_count, Py_ssize_t result_ulength, + CYTHON_UNUSED Py_UCS4 max_char) { +#if CYTHON_USE_UNICODE_INTERNALS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + PyObject *result_uval; + int result_ukind; + Py_ssize_t i, char_pos; + void *result_udata; +#if CYTHON_PEP393_ENABLED + result_uval = PyUnicode_New(result_ulength, max_char); + if (unlikely(!result_uval)) return NULL; + result_ukind = (max_char <= 255) ? PyUnicode_1BYTE_KIND : (max_char <= 65535) ? PyUnicode_2BYTE_KIND : PyUnicode_4BYTE_KIND; + result_udata = PyUnicode_DATA(result_uval); #else - result = PyObject_GetItem(__pyx_d, name); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); + result_uval = PyUnicode_FromUnicode(NULL, result_ulength); + if (unlikely(!result_uval)) return NULL; + result_ukind = sizeof(Py_UNICODE); + result_udata = PyUnicode_AS_UNICODE(result_uval); +#endif + char_pos = 0; + for (i=0; i < value_count; i++) { + int ukind; + Py_ssize_t ulength; + void *udata; + PyObject *uval = PyTuple_GET_ITEM(value_tuple, i); + if (unlikely(__Pyx_PyUnicode_READY(uval))) + goto bad; + ulength = __Pyx_PyUnicode_GET_LENGTH(uval); + if (unlikely(!ulength)) + continue; + if (unlikely(char_pos + ulength < 0)) + goto overflow; + ukind = __Pyx_PyUnicode_KIND(uval); + udata = __Pyx_PyUnicode_DATA(uval); + if (!CYTHON_PEP393_ENABLED || ukind == result_ukind) { + memcpy((char *)result_udata + char_pos * result_ukind, udata, (size_t) (ulength * result_ukind)); + } else { + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030300F0 || defined(_PyUnicode_FastCopyCharacters) + _PyUnicode_FastCopyCharacters(result_uval, char_pos, uval, 0, ulength); + #else + Py_ssize_t j; + for (j=0; j < ulength; j++) { + Py_UCS4 uchar = __Pyx_PyUnicode_READ(ukind, udata, j); + __Pyx_PyUnicode_WRITE(result_ukind, result_udata, char_pos+j, uchar); + } + #endif + } + char_pos += ulength; } - PyErr_Clear(); + return result_uval; +overflow: + PyErr_SetString(PyExc_OverflowError, "join() result is too long for a Python string"); +bad: + Py_DECREF(result_uval); + return NULL; +#else + result_ulength++; + value_count++; + return PyUnicode_Join(__pyx_empty_unicode, value_tuple); #endif - return __Pyx_GetBuiltinName(name); } -/* PyCFunctionFastCall */ -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { - PyCFunctionObject *func = (PyCFunctionObject*)func_obj; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - int flags = PyCFunction_GET_FLAGS(func); - assert(PyCFunction_Check(func)); - assert(METH_FASTCALL == (flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))); - assert(nargs >= 0); - assert(nargs == 0 || args != NULL); - /* _PyCFunction_FastCallDict() must not be called with an exception set, - because it may clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!PyErr_Occurred()); - if ((PY_VERSION_HEX < 0x030700A0) || unlikely(flags & METH_KEYWORDS)) { - return (*((__Pyx_PyCFunctionFastWithKeywords)(void*)meth)) (self, args, nargs, NULL); - } else { - return (*((__Pyx_PyCFunctionFast)(void*)meth)) (self, args, nargs); - } -} -#endif - -/* PyFunctionFastCall */ -#if CYTHON_FAST_PYCALL -static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, - PyObject *globals) { - PyFrameObject *f; - PyThreadState *tstate = __Pyx_PyThreadState_Current; - PyObject **fastlocals; - Py_ssize_t i; - PyObject *result; - assert(globals != NULL); - /* XXX Perhaps we should create a specialized - PyFrame_New() that doesn't take locals, but does - take builtins without sanity checking them. - */ - assert(tstate != NULL); - f = PyFrame_New(tstate, co, globals, NULL); - if (f == NULL) { - return NULL; - } - fastlocals = __Pyx_PyFrame_GetLocalsplus(f); - for (i = 0; i < na; i++) { - Py_INCREF(*args); - fastlocals[i] = *args++; - } - result = PyEval_EvalFrameEx(f,0); - ++tstate->recursion_depth; - Py_DECREF(f); - --tstate->recursion_depth; - return result; -} -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { - PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); - PyObject *globals = PyFunction_GET_GLOBALS(func); - PyObject *argdefs = PyFunction_GET_DEFAULTS(func); - PyObject *closure; -#if PY_MAJOR_VERSION >= 3 - PyObject *kwdefs; -#endif - PyObject *kwtuple, **k; - PyObject **d; - Py_ssize_t nd; - Py_ssize_t nk; - PyObject *result; - assert(kwargs == NULL || PyDict_Check(kwargs)); - nk = kwargs ? PyDict_Size(kwargs) : 0; - if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { - return NULL; - } - if ( -#if PY_MAJOR_VERSION >= 3 - co->co_kwonlyargcount == 0 && -#endif - likely(kwargs == NULL || nk == 0) && - co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { - if (argdefs == NULL && co->co_argcount == nargs) { - result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); - goto done; - } - else if (nargs == 0 && argdefs != NULL - && co->co_argcount == Py_SIZE(argdefs)) { - /* function called with no arguments, but all parameters have - a default value: use default values as arguments .*/ - args = &PyTuple_GET_ITEM(argdefs, 0); - result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); - goto done; - } - } - if (kwargs != NULL) { - Py_ssize_t pos, i; - kwtuple = PyTuple_New(2 * nk); - if (kwtuple == NULL) { - result = NULL; - goto done; - } - k = &PyTuple_GET_ITEM(kwtuple, 0); - pos = i = 0; - while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { - Py_INCREF(k[i]); - Py_INCREF(k[i+1]); - i += 2; - } - nk = i / 2; - } - else { - kwtuple = NULL; - k = NULL; - } - closure = PyFunction_GET_CLOSURE(func); -#if PY_MAJOR_VERSION >= 3 - kwdefs = PyFunction_GET_KW_DEFAULTS(func); -#endif - if (argdefs != NULL) { - d = &PyTuple_GET_ITEM(argdefs, 0); - nd = Py_SIZE(argdefs); - } - else { - d = NULL; - nd = 0; - } -#if PY_MAJOR_VERSION >= 3 - result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, kwdefs, closure); -#else - result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, closure); -#endif - Py_XDECREF(kwtuple); -done: - Py_LeaveRecursiveCall(); - return result; -} -#endif -#endif - /* PyObjectCall */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { @@ -6491,253 +6077,65 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg } #endif -/* PyObjectCall2Args */ -static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { - PyObject *args, *result = NULL; - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[2] = {arg1, arg2}; - return __Pyx_PyFunction_FastCall(function, args, 2); - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[2] = {arg1, arg2}; - return __Pyx_PyCFunction_FastCall(function, args, 2); +/* PyIntBinop */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject* __Pyx_PyInt_AddObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED int inplace) { + #if PY_MAJOR_VERSION < 3 + if (likely(PyInt_CheckExact(op1))) { + const long b = intval; + long x; + long a = PyInt_AS_LONG(op1); + x = (long)((unsigned long)a + b); + if (likely((x^a) >= 0 || (x^b) >= 0)) + return PyInt_FromLong(x); + return PyLong_Type.tp_as_number->nb_add(op1, op2); } #endif - args = PyTuple_New(2); - if (unlikely(!args)) goto done; - Py_INCREF(arg1); - PyTuple_SET_ITEM(args, 0, arg1); - Py_INCREF(arg2); - PyTuple_SET_ITEM(args, 1, arg2); - Py_INCREF(function); - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - Py_DECREF(function); -done: - return result; -} - -/* PyObjectCallMethO */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { - PyObject *self, *result; - PyCFunction cfunc; - cfunc = PyCFunction_GET_FUNCTION(func); - self = PyCFunction_GET_SELF(func); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = cfunc(self, arg); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); - } - return result; -} + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(PyLong_CheckExact(op1))) { + const long b = intval; + long a, x; +#ifdef HAVE_LONG_LONG + const PY_LONG_LONG llb = intval; + PY_LONG_LONG lla, llx; #endif - -/* PyObjectCallOneArg */ -#if CYTHON_COMPILING_IN_CPYTHON -static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_New(1); - if (unlikely(!args)) return NULL; - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 0, arg); - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, &arg, 1); - } + const digit* digits = ((PyLongObject*)op1)->ob_digit; + const Py_ssize_t size = Py_SIZE(op1); + if (likely(__Pyx_sst_abs(size) <= 1)) { + a = likely(size) ? digits[0] : 0; + if (size == -1) a = -a; + } else { + switch (size) { + case -2: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + a = -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; +#ifdef HAVE_LONG_LONG + } else if (8 * sizeof(PY_LONG_LONG) - 1 > 2 * PyLong_SHIFT) { + lla = -(PY_LONG_LONG) (((((unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); + goto long_long; #endif - if (likely(PyCFunction_Check(func))) { - if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { - return __Pyx_PyObject_CallMethO(func, arg); -#if CYTHON_FAST_PYCCALL - } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { - return __Pyx_PyCFunction_FastCall(func, &arg, 1); + } + CYTHON_FALLTHROUGH; + case 2: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + a = (long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; +#ifdef HAVE_LONG_LONG + } else if (8 * sizeof(PY_LONG_LONG) - 1 > 2 * PyLong_SHIFT) { + lla = (PY_LONG_LONG) (((((unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); + goto long_long; #endif - } - } - return __Pyx__PyObject_CallOneArg(func, arg); -} -#else -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_Pack(1, arg); - if (unlikely(!args)) return NULL; - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -#endif - -/* ArgTypeTest */ -static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) -{ - if (unlikely(!type)) { - PyErr_SetString(PyExc_SystemError, "Missing type object"); - return 0; - } - else if (exact) { - #if PY_MAJOR_VERSION == 2 - if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; - #endif - } - else { - if (likely(__Pyx_TypeCheck(obj, type))) return 1; - } - PyErr_Format(PyExc_TypeError, - "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", - name, type->tp_name, Py_TYPE(obj)->tp_name); - return 0; -} - -/* unicode_iter */ -static CYTHON_INLINE int __Pyx_init_unicode_iteration( - PyObject* ustring, Py_ssize_t *length, void** data, int *kind) { -#if CYTHON_PEP393_ENABLED - if (unlikely(__Pyx_PyUnicode_READY(ustring) < 0)) return -1; - *kind = PyUnicode_KIND(ustring); - *length = PyUnicode_GET_LENGTH(ustring); - *data = PyUnicode_DATA(ustring); -#else - *kind = 0; - *length = PyUnicode_GET_SIZE(ustring); - *data = (void*)PyUnicode_AS_UNICODE(ustring); -#endif - return 0; -} - -/* JoinPyUnicode */ -static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_count, Py_ssize_t result_ulength, - CYTHON_UNUSED Py_UCS4 max_char) { -#if CYTHON_USE_UNICODE_INTERNALS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - PyObject *result_uval; - int result_ukind; - Py_ssize_t i, char_pos; - void *result_udata; -#if CYTHON_PEP393_ENABLED - result_uval = PyUnicode_New(result_ulength, max_char); - if (unlikely(!result_uval)) return NULL; - result_ukind = (max_char <= 255) ? PyUnicode_1BYTE_KIND : (max_char <= 65535) ? PyUnicode_2BYTE_KIND : PyUnicode_4BYTE_KIND; - result_udata = PyUnicode_DATA(result_uval); -#else - result_uval = PyUnicode_FromUnicode(NULL, result_ulength); - if (unlikely(!result_uval)) return NULL; - result_ukind = sizeof(Py_UNICODE); - result_udata = PyUnicode_AS_UNICODE(result_uval); -#endif - char_pos = 0; - for (i=0; i < value_count; i++) { - int ukind; - Py_ssize_t ulength; - void *udata; - PyObject *uval = PyTuple_GET_ITEM(value_tuple, i); - if (unlikely(__Pyx_PyUnicode_READY(uval))) - goto bad; - ulength = __Pyx_PyUnicode_GET_LENGTH(uval); - if (unlikely(!ulength)) - continue; - if (unlikely(char_pos + ulength < 0)) - goto overflow; - ukind = __Pyx_PyUnicode_KIND(uval); - udata = __Pyx_PyUnicode_DATA(uval); - if (!CYTHON_PEP393_ENABLED || ukind == result_ukind) { - memcpy((char *)result_udata + char_pos * result_ukind, udata, (size_t) (ulength * result_ukind)); - } else { - #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030300F0 || defined(_PyUnicode_FastCopyCharacters) - _PyUnicode_FastCopyCharacters(result_uval, char_pos, uval, 0, ulength); - #else - Py_ssize_t j; - for (j=0; j < ulength; j++) { - Py_UCS4 uchar = __Pyx_PyUnicode_READ(ukind, udata, j); - __Pyx_PyUnicode_WRITE(result_ukind, result_udata, char_pos+j, uchar); - } - #endif - } - char_pos += ulength; - } - return result_uval; -overflow: - PyErr_SetString(PyExc_OverflowError, "join() result is too long for a Python string"); -bad: - Py_DECREF(result_uval); - return NULL; -#else - result_ulength++; - value_count++; - return PyUnicode_Join(__pyx_empty_unicode, value_tuple); -#endif -} - -/* PyIntBinop */ -#if !CYTHON_COMPILING_IN_PYPY -static PyObject* __Pyx_PyInt_AddObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED int inplace) { - #if PY_MAJOR_VERSION < 3 - if (likely(PyInt_CheckExact(op1))) { - const long b = intval; - long x; - long a = PyInt_AS_LONG(op1); - x = (long)((unsigned long)a + b); - if (likely((x^a) >= 0 || (x^b) >= 0)) - return PyInt_FromLong(x); - return PyLong_Type.tp_as_number->nb_add(op1, op2); - } - #endif - #if CYTHON_USE_PYLONG_INTERNALS - if (likely(PyLong_CheckExact(op1))) { - const long b = intval; - long a, x; -#ifdef HAVE_LONG_LONG - const PY_LONG_LONG llb = intval; - PY_LONG_LONG lla, llx; -#endif - const digit* digits = ((PyLongObject*)op1)->ob_digit; - const Py_ssize_t size = Py_SIZE(op1); - if (likely(__Pyx_sst_abs(size) <= 1)) { - a = likely(size) ? digits[0] : 0; - if (size == -1) a = -a; - } else { - switch (size) { - case -2: - if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - a = -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; -#ifdef HAVE_LONG_LONG - } else if (8 * sizeof(PY_LONG_LONG) - 1 > 2 * PyLong_SHIFT) { - lla = -(PY_LONG_LONG) (((((unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); - goto long_long; -#endif - } - CYTHON_FALLTHROUGH; - case 2: - if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - a = (long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; -#ifdef HAVE_LONG_LONG - } else if (8 * sizeof(PY_LONG_LONG) - 1 > 2 * PyLong_SHIFT) { - lla = (PY_LONG_LONG) (((((unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); - goto long_long; -#endif - } - CYTHON_FALLTHROUGH; - case -3: - if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - a = -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; -#ifdef HAVE_LONG_LONG - } else if (8 * sizeof(PY_LONG_LONG) - 1 > 3 * PyLong_SHIFT) { - lla = -(PY_LONG_LONG) (((((((unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); - goto long_long; + } + CYTHON_FALLTHROUGH; + case -3: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + a = -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; +#ifdef HAVE_LONG_LONG + } else if (8 * sizeof(PY_LONG_LONG) - 1 > 3 * PyLong_SHIFT) { + lla = -(PY_LONG_LONG) (((((((unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); + goto long_long; #endif } CYTHON_FALLTHROUGH; @@ -6993,162 +6391,364 @@ static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject } #endif -/* RaiseException */ -#if PY_MAJOR_VERSION < 3 -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, - CYTHON_UNUSED PyObject *cause) { - __Pyx_PyThreadState_declare - Py_XINCREF(type); - if (!value || value == Py_None) - value = NULL; - else - Py_INCREF(value); - if (!tb || tb == Py_None) - tb = NULL; - else { - Py_INCREF(tb); - if (!PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto raise_error; - } +/* RaiseException */ +#if PY_MAJOR_VERSION < 3 +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, + CYTHON_UNUSED PyObject *cause) { + __Pyx_PyThreadState_declare + Py_XINCREF(type); + if (!value || value == Py_None) + value = NULL; + else + Py_INCREF(value); + if (!tb || tb == Py_None) + tb = NULL; + else { + Py_INCREF(tb); + if (!PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto raise_error; + } + } + if (PyType_Check(type)) { +#if CYTHON_COMPILING_IN_PYPY + if (!value) { + Py_INCREF(Py_None); + value = Py_None; + } +#endif + PyErr_NormalizeException(&type, &value, &tb); + } else { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto raise_error; + } + value = type; + type = (PyObject*) Py_TYPE(type); + Py_INCREF(type); + if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto raise_error; + } + } + __Pyx_PyThreadState_assign + __Pyx_ErrRestore(type, value, tb); + return; +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(tb); + return; +} +#else +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if CYTHON_COMPILING_IN_PYPY + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#else + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} +#endif + +/* PyCFunctionFastCall */ +#if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { + PyCFunctionObject *func = (PyCFunctionObject*)func_obj; + PyCFunction meth = PyCFunction_GET_FUNCTION(func); + PyObject *self = PyCFunction_GET_SELF(func); + int flags = PyCFunction_GET_FLAGS(func); + assert(PyCFunction_Check(func)); + assert(METH_FASTCALL == (flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))); + assert(nargs >= 0); + assert(nargs == 0 || args != NULL); + /* _PyCFunction_FastCallDict() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!PyErr_Occurred()); + if ((PY_VERSION_HEX < 0x030700A0) || unlikely(flags & METH_KEYWORDS)) { + return (*((__Pyx_PyCFunctionFastWithKeywords)(void*)meth)) (self, args, nargs, NULL); + } else { + return (*((__Pyx_PyCFunctionFast)(void*)meth)) (self, args, nargs); + } +} +#endif + +/* PyFunctionFastCall */ +#if CYTHON_FAST_PYCALL +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = __Pyx_PyFrame_GetLocalsplus(f); + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; +#if PY_MAJOR_VERSION >= 3 + PyObject *kwdefs; +#endif + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { + return NULL; + } + if ( +#if PY_MAJOR_VERSION >= 3 + co->co_kwonlyargcount == 0 && +#endif + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); +#if PY_MAJOR_VERSION >= 3 + kwdefs = PyFunction_GET_KW_DEFAULTS(func); +#endif + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } +#if PY_MAJOR_VERSION >= 3 + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); +#else + result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, closure); +#endif + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif +#endif + +/* PyObjectCallMethO */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = PyCFunction_GET_FUNCTION(func); + self = PyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallOneArg */ +#if CYTHON_COMPILING_IN_CPYTHON +static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_New(1); + if (unlikely(!args)) return NULL; + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 0, arg); + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, &arg, 1); } - if (PyType_Check(type)) { -#if CYTHON_COMPILING_IN_PYPY - if (!value) { - Py_INCREF(Py_None); - value = Py_None; - } #endif - PyErr_NormalizeException(&type, &value, &tb); - } else { - if (value) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto raise_error; - } - value = type; - type = (PyObject*) Py_TYPE(type); - Py_INCREF(type); - if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto raise_error; + if (likely(PyCFunction_Check(func))) { + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, arg); +#if CYTHON_FAST_PYCCALL + } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { + return __Pyx_PyCFunction_FastCall(func, &arg, 1); +#endif } } - __Pyx_PyThreadState_assign - __Pyx_ErrRestore(type, value, tb); - return; -raise_error: - Py_XDECREF(value); - Py_XDECREF(type); - Py_XDECREF(tb); - return; + return __Pyx__PyObject_CallOneArg(func, arg); } #else -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { - PyObject* owned_instance = NULL; - if (tb == Py_None) { - tb = 0; - } else if (tb && !PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto bad; - } - if (value == Py_None) - value = 0; - if (PyExceptionInstance_Check(type)) { - if (value) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto bad; - } - value = type; - type = (PyObject*) Py_TYPE(value); - } else if (PyExceptionClass_Check(type)) { - PyObject *instance_class = NULL; - if (value && PyExceptionInstance_Check(value)) { - instance_class = (PyObject*) Py_TYPE(value); - if (instance_class != type) { - int is_subclass = PyObject_IsSubclass(instance_class, type); - if (!is_subclass) { - instance_class = NULL; - } else if (unlikely(is_subclass == -1)) { - goto bad; - } else { - type = instance_class; - } - } - } - if (!instance_class) { - PyObject *args; - if (!value) - args = PyTuple_New(0); - else if (PyTuple_Check(value)) { - Py_INCREF(value); - args = value; - } else - args = PyTuple_Pack(1, value); - if (!args) - goto bad; - owned_instance = PyObject_Call(type, args, NULL); - Py_DECREF(args); - if (!owned_instance) - goto bad; - value = owned_instance; - if (!PyExceptionInstance_Check(value)) { - PyErr_Format(PyExc_TypeError, - "calling %R should have returned an instance of " - "BaseException, not %R", - type, Py_TYPE(value)); - goto bad; - } - } - } else { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto bad; - } - if (cause) { - PyObject *fixed_cause; - if (cause == Py_None) { - fixed_cause = NULL; - } else if (PyExceptionClass_Check(cause)) { - fixed_cause = PyObject_CallObject(cause, NULL); - if (fixed_cause == NULL) - goto bad; - } else if (PyExceptionInstance_Check(cause)) { - fixed_cause = cause; - Py_INCREF(fixed_cause); - } else { - PyErr_SetString(PyExc_TypeError, - "exception causes must derive from " - "BaseException"); - goto bad; - } - PyException_SetCause(value, fixed_cause); - } - PyErr_SetObject(type, value); - if (tb) { -#if CYTHON_COMPILING_IN_PYPY - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); - Py_INCREF(tb); - PyErr_Restore(tmp_type, tmp_value, tb); - Py_XDECREF(tmp_tb); -#else - PyThreadState *tstate = __Pyx_PyThreadState_Current; - PyObject* tmp_tb = tstate->curexc_traceback; - if (tb != tmp_tb) { - Py_INCREF(tb); - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_tb); - } -#endif - } -bad: - Py_XDECREF(owned_instance); - return; +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_Pack(1, arg); + if (unlikely(!args)) return NULL; + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; } #endif @@ -8177,6 +7777,41 @@ static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { return value; } +/* GetModuleGlobalName */ +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ + PyObject *result; +#if !CYTHON_AVOID_BORROWED_REFS +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 + result = _PyDict_GetItem_KnownHash(__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } else if (unlikely(PyErr_Occurred())) { + return NULL; + } +#else + result = PyDict_GetItem(__pyx_d, name); + __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } +#endif +#else + result = PyObject_GetItem(__pyx_d, name); + __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); +} + /* CLineInTraceback */ #ifndef CYTHON_CLINE_IN_TRACEBACK static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { @@ -8438,24 +8073,24 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { } /* CIntToPy */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value) { - const unsigned int neg_one = (unsigned int) ((unsigned int) 0 - (unsigned int) 1), const_zero = (unsigned int) 0; +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint32_t(uint32_t value) { + const uint32_t neg_one = (uint32_t) ((uint32_t) 0 - (uint32_t) 1), const_zero = (uint32_t) 0; const int is_unsigned = neg_one > const_zero; if (is_unsigned) { - if (sizeof(unsigned int) < sizeof(long)) { + if (sizeof(uint32_t) < sizeof(long)) { return PyInt_FromLong((long) value); - } else if (sizeof(unsigned int) <= sizeof(unsigned long)) { + } else if (sizeof(uint32_t) <= sizeof(unsigned long)) { return PyLong_FromUnsignedLong((unsigned long) value); #ifdef HAVE_LONG_LONG - } else if (sizeof(unsigned int) <= sizeof(unsigned PY_LONG_LONG)) { + } else if (sizeof(uint32_t) <= sizeof(unsigned PY_LONG_LONG)) { return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); #endif } } else { - if (sizeof(unsigned int) <= sizeof(long)) { + if (sizeof(uint32_t) <= sizeof(long)) { return PyInt_FromLong((long) value); #ifdef HAVE_LONG_LONG - } else if (sizeof(unsigned int) <= sizeof(PY_LONG_LONG)) { + } else if (sizeof(uint32_t) <= sizeof(PY_LONG_LONG)) { return PyLong_FromLongLong((PY_LONG_LONG) value); #endif } @@ -8463,30 +8098,30 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value) { int one = 1; int little = (int)*(unsigned char *)&one; unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(unsigned int), + return _PyLong_FromByteArray(bytes, sizeof(uint32_t), little, !is_unsigned); } } /* CIntToPy */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint32_t(uint32_t value) { - const uint32_t neg_one = (uint32_t) ((uint32_t) 0 - (uint32_t) 1), const_zero = (uint32_t) 0; +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value) { + const unsigned int neg_one = (unsigned int) ((unsigned int) 0 - (unsigned int) 1), const_zero = (unsigned int) 0; const int is_unsigned = neg_one > const_zero; if (is_unsigned) { - if (sizeof(uint32_t) < sizeof(long)) { + if (sizeof(unsigned int) < sizeof(long)) { return PyInt_FromLong((long) value); - } else if (sizeof(uint32_t) <= sizeof(unsigned long)) { + } else if (sizeof(unsigned int) <= sizeof(unsigned long)) { return PyLong_FromUnsignedLong((unsigned long) value); #ifdef HAVE_LONG_LONG - } else if (sizeof(uint32_t) <= sizeof(unsigned PY_LONG_LONG)) { + } else if (sizeof(unsigned int) <= sizeof(unsigned PY_LONG_LONG)) { return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); #endif } } else { - if (sizeof(uint32_t) <= sizeof(long)) { + if (sizeof(unsigned int) <= sizeof(long)) { return PyInt_FromLong((long) value); #ifdef HAVE_LONG_LONG - } else if (sizeof(uint32_t) <= sizeof(PY_LONG_LONG)) { + } else if (sizeof(unsigned int) <= sizeof(PY_LONG_LONG)) { return PyLong_FromLongLong((PY_LONG_LONG) value); #endif } @@ -8494,7 +8129,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint32_t(uint32_t value) { { int one = 1; int little = (int)*(unsigned char *)&one; unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(uint32_t), + return _PyLong_FromByteArray(bytes, sizeof(unsigned int), little, !is_unsigned); } } diff --git a/gensim/models/_utils_any2vec.pyx b/gensim/models/_utils_any2vec.pyx index f5672c0b14..ce41eefb7a 100644 --- a/gensim/models/_utils_any2vec.pyx +++ b/gensim/models/_utils_any2vec.pyx @@ -22,34 +22,6 @@ cdef _byte_to_int_py2(b): _byte_to_int = _byte_to_int_py2 if PY2 else _byte_to_int_py3 -cpdef ft_hash(unicode string): - """Calculate hash based on `string`. - Reproduce `hash method from Facebook fastText implementation - `_. - - This function is expensive because it encodes each input string as UTF-8, - and this can accumulate to a high computational cost if there are many - strings. If possible, convert your strings to bytes and use the faster - :py:func:`ft_hash_bytes` function instead. - - Parameters - ---------- - string : unicode - The string whose hash needs to be calculated. - - Returns - ------- - unsigned int - The hash of the string. - - """ - cdef unsigned int h = 2166136261 - for c in string.encode("utf-8"): - h = np.uint32(h ^ np.uint32(np.int8(_byte_to_int(c)))) - h = np.uint32(h * np.uint32(16777619)) - return h - - cpdef ft_hash_bytes(bytes bytez): """Calculate hash based on `bytez`. Reproduce `hash method from Facebook fastText implementation From 8144e20f6146ac1d8eac7d5cbc6e88e3f88b82df Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Sun, 20 Jan 2019 19:30:31 +1100 Subject: [PATCH 06/32] review response: update docstring --- gensim/models/_fasttext_bin.py | 2 +- gensim/models/fasttext.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/gensim/models/_fasttext_bin.py b/gensim/models/_fasttext_bin.py index 2cbc530a96..ab88dbffda 100644 --- a/gensim/models/_fasttext_bin.py +++ b/gensim/models/_fasttext_bin.py @@ -231,7 +231,7 @@ def load(fin, encoding='utf-8', full_model=True): The encoding to use for decoding text full_model : boolean, optional If False, skips loading the hidden output matrix. This saves a fair bit - of time, but prevents training continuation. + of CPU time and RAM, but prevents training continuation. Returns ------- diff --git a/gensim/models/fasttext.py b/gensim/models/fasttext.py index b51494578c..1e0d442e65 100644 --- a/gensim/models/fasttext.py +++ b/gensim/models/fasttext.py @@ -779,7 +779,7 @@ def load_fasttext_format(cls, model_file, encoding='utf8', full_model=True): Specifies the file encoding. full_model : boolean, optional If False, skips loading the hidden output matrix. This saves a fair bit - of time, but prevents training continuation. + of CPU time and RAM, but prevents training continuation. Returns ------- @@ -983,12 +983,13 @@ def _load_fasttext_format(model_file, encoding='utf-8', full_model=True): Specifies the file encoding. full_model : boolean, optional If False, skips loading the hidden output matrix. This saves a fair bit - of time, but prevents training continuation. + of CPU time and RAM, but prevents training continuation. Returns ------- :class: `~gensim.models.fasttext.FastText` The loaded model. + """ if not model_file.endswith('.bin'): model_file += '.bin' From d735099720f64ec78c26a48560a4c5da008a5e82 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Mon, 21 Jan 2019 19:18:11 +1100 Subject: [PATCH 07/32] attempt to hack around appveyor Py2.7 build missing stdint.h --- gensim/models/_utils_any2vec.c | 206 +++++++++++++++---------------- gensim/models/_utils_any2vec.pyx | 5 +- gensim/models/stdint_wrapper.h | 17 +++ 3 files changed, 123 insertions(+), 105 deletions(-) create mode 100644 gensim/models/stdint_wrapper.h diff --git a/gensim/models/_utils_any2vec.c b/gensim/models/_utils_any2vec.c index 3f498f12e2..67eee33698 100644 --- a/gensim/models/_utils_any2vec.c +++ b/gensim/models/_utils_any2vec.c @@ -608,7 +608,7 @@ static CYTHON_INLINE float __PYX_NAN() { #define __PYX_HAVE__gensim__models___utils_any2vec #define __PYX_HAVE_API__gensim__models___utils_any2vec /* Early includes */ -#include +#include "stdint_wrapper.h" #include #include #include "numpy/arrayobject.h" @@ -1691,8 +1691,6 @@ static int __Pyx_check_binary_version(void); static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); -/* Module declarations from 'libc.stdint' */ - /* Module declarations from 'cpython.buffer' */ /* Module declarations from 'libc.string' */ @@ -1824,7 +1822,7 @@ static PyObject *__pyx_tuple__10; static PyObject *__pyx_codeobj__11; /* Late includes */ -/* "gensim/models/_utils_any2vec.pyx":16 +/* "gensim/models/_utils_any2vec.pyx":19 * * * cdef _byte_to_int_py3(b): # <<<<<<<<<<<<<< @@ -1837,7 +1835,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3(PyObj __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("_byte_to_int_py3", 0); - /* "gensim/models/_utils_any2vec.pyx":17 + /* "gensim/models/_utils_any2vec.pyx":20 * * cdef _byte_to_int_py3(b): * return b # <<<<<<<<<<<<<< @@ -1849,7 +1847,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3(PyObj __pyx_r = __pyx_v_b; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":16 + /* "gensim/models/_utils_any2vec.pyx":19 * * * cdef _byte_to_int_py3(b): # <<<<<<<<<<<<<< @@ -1864,7 +1862,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3(PyObj return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":19 +/* "gensim/models/_utils_any2vec.pyx":22 * return b * * cdef _byte_to_int_py2(b): # <<<<<<<<<<<<<< @@ -1879,7 +1877,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObj PyObject *__pyx_t_2 = NULL; __Pyx_RefNannySetupContext("_byte_to_int_py2", 0); - /* "gensim/models/_utils_any2vec.pyx":20 + /* "gensim/models/_utils_any2vec.pyx":23 * * cdef _byte_to_int_py2(b): * return ord(b) # <<<<<<<<<<<<<< @@ -1887,14 +1885,14 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObj * _byte_to_int = _byte_to_int_py2 if PY2 else _byte_to_int_py3 */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_Ord(__pyx_v_b); if (unlikely(__pyx_t_1 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 20, __pyx_L1_error) - __pyx_t_2 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 20, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Ord(__pyx_v_b); if (unlikely(__pyx_t_1 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 23, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 23, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":19 + /* "gensim/models/_utils_any2vec.pyx":22 * return b * * cdef _byte_to_int_py2(b): # <<<<<<<<<<<<<< @@ -1913,7 +1911,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObj return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":25 +/* "gensim/models/_utils_any2vec.pyx":28 * * * cpdef ft_hash_bytes(bytes bytez): # <<<<<<<<<<<<<< @@ -1935,7 +1933,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject PyObject *__pyx_t_6 = NULL; __Pyx_RefNannySetupContext("ft_hash_bytes", 0); - /* "gensim/models/_utils_any2vec.pyx":41 + /* "gensim/models/_utils_any2vec.pyx":44 * * """ * cdef uint32_t h = 2166136261 # <<<<<<<<<<<<<< @@ -1944,7 +1942,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject */ __pyx_v_h = 0x811C9DC5; - /* "gensim/models/_utils_any2vec.pyx":44 + /* "gensim/models/_utils_any2vec.pyx":47 * cdef char b * * for b in bytez: # <<<<<<<<<<<<<< @@ -1953,7 +1951,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject */ if (unlikely(__pyx_v_bytez == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); - __PYX_ERR(0, 44, __pyx_L1_error) + __PYX_ERR(0, 47, __pyx_L1_error) } __Pyx_INCREF(__pyx_v_bytez); __pyx_t_1 = __pyx_v_bytez; @@ -1963,7 +1961,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject __pyx_t_2 = __pyx_t_5; __pyx_v_b = (__pyx_t_2[0]); - /* "gensim/models/_utils_any2vec.pyx":45 + /* "gensim/models/_utils_any2vec.pyx":48 * * for b in bytez: * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss # <<<<<<<<<<<<<< @@ -1972,7 +1970,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject */ __pyx_v_h = (__pyx_v_h ^ ((uint32_t)((int8_t)__pyx_v_b))); - /* "gensim/models/_utils_any2vec.pyx":46 + /* "gensim/models/_utils_any2vec.pyx":49 * for b in bytez: * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss * h = h * 16777619 # <<<<<<<<<<<<<< @@ -1983,7 +1981,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":47 + /* "gensim/models/_utils_any2vec.pyx":50 * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss * h = h * 16777619 * return h # <<<<<<<<<<<<<< @@ -1991,13 +1989,13 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_6 = __Pyx_PyInt_From_uint32_t(__pyx_v_h); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 47, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyInt_From_uint32_t(__pyx_v_h); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 50, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_r = __pyx_t_6; __pyx_t_6 = 0; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":25 + /* "gensim/models/_utils_any2vec.pyx":28 * * * cpdef ft_hash_bytes(bytes bytez): # <<<<<<<<<<<<<< @@ -2024,7 +2022,7 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash_bytes(PyObje PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("ft_hash_bytes (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_bytez), (&PyBytes_Type), 1, "bytez", 1))) __PYX_ERR(0, 25, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_bytez), (&PyBytes_Type), 1, "bytez", 1))) __PYX_ERR(0, 28, __pyx_L1_error) __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(__pyx_self, ((PyObject*)__pyx_v_bytez)); /* function exit code */ @@ -2042,7 +2040,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(CYTHON_ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("ft_hash_bytes", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(__pyx_v_bytez, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 25, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(__pyx_v_bytez, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 28, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -2059,7 +2057,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(CYTHON_ return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":50 +/* "gensim/models/_utils_any2vec.pyx":53 * * * cpdef ft_hash_broken(unicode string): # <<<<<<<<<<<<<< @@ -2084,7 +2082,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec long __pyx_t_9; __Pyx_RefNannySetupContext("ft_hash_broken", 0); - /* "gensim/models/_utils_any2vec.pyx":67 + /* "gensim/models/_utils_any2vec.pyx":70 * * """ * cdef unsigned int h = 2166136261 # <<<<<<<<<<<<<< @@ -2093,7 +2091,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec */ __pyx_v_h = 0x811C9DC5; - /* "gensim/models/_utils_any2vec.pyx":68 + /* "gensim/models/_utils_any2vec.pyx":71 * """ * cdef unsigned int h = 2166136261 * for c in string: # <<<<<<<<<<<<<< @@ -2102,29 +2100,29 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec */ if (unlikely(__pyx_v_string == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); - __PYX_ERR(0, 68, __pyx_L1_error) + __PYX_ERR(0, 71, __pyx_L1_error) } __Pyx_INCREF(__pyx_v_string); __pyx_t_1 = __pyx_v_string; - __pyx_t_6 = __Pyx_init_unicode_iteration(__pyx_t_1, (&__pyx_t_3), (&__pyx_t_4), (&__pyx_t_5)); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 68, __pyx_L1_error) + __pyx_t_6 = __Pyx_init_unicode_iteration(__pyx_t_1, (&__pyx_t_3), (&__pyx_t_4), (&__pyx_t_5)); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 71, __pyx_L1_error) for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_3; __pyx_t_7++) { __pyx_t_2 = __pyx_t_7; - __pyx_t_8 = PyUnicode_FromOrdinal(__Pyx_PyUnicode_READ(__pyx_t_5, __pyx_t_4, __pyx_t_2)); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 68, __pyx_L1_error) + __pyx_t_8 = PyUnicode_FromOrdinal(__Pyx_PyUnicode_READ(__pyx_t_5, __pyx_t_4, __pyx_t_2)); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 71, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_XDECREF_SET(__pyx_v_c, __pyx_t_8); __pyx_t_8 = 0; - /* "gensim/models/_utils_any2vec.pyx":69 + /* "gensim/models/_utils_any2vec.pyx":72 * cdef unsigned int h = 2166136261 * for c in string: * h ^= ord(c) # <<<<<<<<<<<<<< * h *= 16777619 * return h */ - __pyx_t_9 = __Pyx_PyObject_Ord(__pyx_v_c); if (unlikely(__pyx_t_9 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 69, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyObject_Ord(__pyx_v_c); if (unlikely(__pyx_t_9 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 72, __pyx_L1_error) __pyx_v_h = (__pyx_v_h ^ __pyx_t_9); - /* "gensim/models/_utils_any2vec.pyx":70 + /* "gensim/models/_utils_any2vec.pyx":73 * for c in string: * h ^= ord(c) * h *= 16777619 # <<<<<<<<<<<<<< @@ -2135,7 +2133,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":71 + /* "gensim/models/_utils_any2vec.pyx":74 * h ^= ord(c) * h *= 16777619 * return h # <<<<<<<<<<<<<< @@ -2143,13 +2141,13 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_8 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 71, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 74, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __pyx_r = __pyx_t_8; __pyx_t_8 = 0; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":50 + /* "gensim/models/_utils_any2vec.pyx":53 * * * cpdef ft_hash_broken(unicode string): # <<<<<<<<<<<<<< @@ -2177,7 +2175,7 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken(PyObj PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("ft_hash_broken (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_string), (&PyUnicode_Type), 1, "string", 1))) __PYX_ERR(0, 50, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_string), (&PyUnicode_Type), 1, "string", 1))) __PYX_ERR(0, 53, __pyx_L1_error) __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(__pyx_self, ((PyObject*)__pyx_v_string)); /* function exit code */ @@ -2195,7 +2193,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHO PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("ft_hash_broken", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(__pyx_v_string, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 50, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(__pyx_v_string, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 53, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -2212,7 +2210,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHO return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":74 +/* "gensim/models/_utils_any2vec.pyx":77 * * * cpdef compute_ngrams(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -2244,14 +2242,14 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec int __pyx_t_14; __Pyx_RefNannySetupContext("compute_ngrams", 0); - /* "gensim/models/_utils_any2vec.pyx":92 + /* "gensim/models/_utils_any2vec.pyx":95 * * """ * cdef unicode extended_word = f'<{word}>' # <<<<<<<<<<<<<< * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): */ - __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 95, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = 0; __pyx_t_3 = 127; @@ -2259,7 +2257,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __pyx_t_2 += 1; __Pyx_GIVEREF(__pyx_kp_u_); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_kp_u_); - __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_word, __pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_word, __pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 95, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = (__Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) > __pyx_t_3) ? __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) : __pyx_t_3; __pyx_t_2 += __Pyx_PyUnicode_GET_LENGTH(__pyx_t_4); @@ -2270,43 +2268,43 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __pyx_t_2 += 1; __Pyx_GIVEREF(__pyx_kp_u__2); PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_kp_u__2); - __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_1, 3, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_1, 3, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 95, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_extended_word = ((PyObject*)__pyx_t_4); __pyx_t_4 = 0; - /* "gensim/models/_utils_any2vec.pyx":93 + /* "gensim/models/_utils_any2vec.pyx":96 * """ * cdef unicode extended_word = f'<{word}>' * ngrams = [] # <<<<<<<<<<<<<< * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): */ - __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 93, __pyx_L1_error) + __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 96, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_v_ngrams = ((PyObject*)__pyx_t_4); __pyx_t_4 = 0; - /* "gensim/models/_utils_any2vec.pyx":94 + /* "gensim/models/_utils_any2vec.pyx":97 * cdef unicode extended_word = f'<{word}>' * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): # <<<<<<<<<<<<<< * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) */ - __pyx_t_4 = __Pyx_PyInt_From_unsigned_int(__pyx_v_min_n); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 94, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyInt_From_unsigned_int(__pyx_v_min_n); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 97, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = __pyx_v_max_n; - __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 94, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 97, __pyx_L1_error) if (((__pyx_t_5 < __pyx_t_2) != 0)) { __pyx_t_6 = __pyx_t_5; } else { __pyx_t_6 = __pyx_t_2; } - __pyx_t_1 = PyInt_FromSsize_t((__pyx_t_6 + 1)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 94, __pyx_L1_error) + __pyx_t_1 = PyInt_FromSsize_t((__pyx_t_6 + 1)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 97, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 94, __pyx_L1_error) + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 97, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_4); @@ -2314,16 +2312,16 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_1); __pyx_t_4 = 0; __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 94, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 97, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (likely(PyList_CheckExact(__pyx_t_1)) || PyTuple_CheckExact(__pyx_t_1)) { __pyx_t_7 = __pyx_t_1; __Pyx_INCREF(__pyx_t_7); __pyx_t_6 = 0; __pyx_t_8 = NULL; } else { - __pyx_t_6 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 94, __pyx_L1_error) + __pyx_t_6 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 97, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_8 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 94, __pyx_L1_error) + __pyx_t_8 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 97, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; for (;;) { @@ -2331,17 +2329,17 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec if (likely(PyList_CheckExact(__pyx_t_7))) { if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_7)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 94, __pyx_L1_error) + __pyx_t_1 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 97, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 94, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 97, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } else { if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_7)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 94, __pyx_L1_error) + __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 97, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 94, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 97, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } @@ -2351,7 +2349,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(0, 94, __pyx_L1_error) + else __PYX_ERR(0, 97, __pyx_L1_error) } break; } @@ -2360,23 +2358,23 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __Pyx_XDECREF_SET(__pyx_v_ngram_length, __pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":95 + /* "gensim/models/_utils_any2vec.pyx":98 * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): # <<<<<<<<<<<<<< * ngrams.append(extended_word[i:i + ngram_length]) * return ngrams */ - __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 95, __pyx_L1_error) - __pyx_t_1 = PyInt_FromSsize_t(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 95, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_t_1 = PyInt_FromSsize_t(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 98, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_4 = PyNumber_Subtract(__pyx_t_1, __pyx_v_ngram_length); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 95, __pyx_L1_error) + __pyx_t_4 = PyNumber_Subtract(__pyx_t_1, __pyx_v_ngram_length); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 98, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyInt_AddObjC(__pyx_t_4, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 95, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_AddObjC(__pyx_t_4, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 98, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 95, __pyx_L1_error) + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 98, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_INCREF(__pyx_int_0); __Pyx_GIVEREF(__pyx_int_0); @@ -2384,16 +2382,16 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 95, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 98, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (likely(PyList_CheckExact(__pyx_t_1)) || PyTuple_CheckExact(__pyx_t_1)) { __pyx_t_4 = __pyx_t_1; __Pyx_INCREF(__pyx_t_4); __pyx_t_2 = 0; __pyx_t_9 = NULL; } else { - __pyx_t_2 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 95, __pyx_L1_error) + __pyx_t_2 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 98, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_9 = Py_TYPE(__pyx_t_4)->tp_iternext; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 95, __pyx_L1_error) + __pyx_t_9 = Py_TYPE(__pyx_t_4)->tp_iternext; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 98, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; for (;;) { @@ -2401,17 +2399,17 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec if (likely(PyList_CheckExact(__pyx_t_4))) { if (__pyx_t_2 >= PyList_GET_SIZE(__pyx_t_4)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 95, __pyx_L1_error) + __pyx_t_1 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 98, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 95, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 98, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } else { if (__pyx_t_2 >= PyTuple_GET_SIZE(__pyx_t_4)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 95, __pyx_L1_error) + __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 98, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 95, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 98, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } @@ -2421,7 +2419,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(0, 95, __pyx_L1_error) + else __PYX_ERR(0, 98, __pyx_L1_error) } break; } @@ -2430,7 +2428,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __Pyx_XDECREF_SET(__pyx_v_i, __pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":96 + /* "gensim/models/_utils_any2vec.pyx":99 * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) # <<<<<<<<<<<<<< @@ -2442,26 +2440,26 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec if (__pyx_t_11) { __pyx_t_10 = 0; } else { - __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 96, __pyx_L1_error) + __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 99, __pyx_L1_error) __pyx_t_10 = __pyx_t_12; } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyNumber_Add(__pyx_v_i, __pyx_v_ngram_length); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 96, __pyx_L1_error) + __pyx_t_1 = PyNumber_Add(__pyx_v_i, __pyx_v_ngram_length); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 99, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_11 = (__pyx_t_1 == Py_None); if (__pyx_t_11) { __pyx_t_12 = PY_SSIZE_T_MAX; } else { - __pyx_t_13 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_13 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 96, __pyx_L1_error) + __pyx_t_13 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_13 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 99, __pyx_L1_error) __pyx_t_12 = __pyx_t_13; } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyUnicode_Substring(__pyx_v_extended_word, __pyx_t_10, __pyx_t_12); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 96, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyUnicode_Substring(__pyx_v_extended_word, __pyx_t_10, __pyx_t_12); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 99, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_1); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 96, __pyx_L1_error) + __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_1); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 99, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":95 + /* "gensim/models/_utils_any2vec.pyx":98 * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): # <<<<<<<<<<<<<< @@ -2471,7 +2469,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec } __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "gensim/models/_utils_any2vec.pyx":94 + /* "gensim/models/_utils_any2vec.pyx":97 * cdef unicode extended_word = f'<{word}>' * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): # <<<<<<<<<<<<<< @@ -2481,7 +2479,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - /* "gensim/models/_utils_any2vec.pyx":97 + /* "gensim/models/_utils_any2vec.pyx":100 * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) * return ngrams # <<<<<<<<<<<<<< @@ -2491,7 +2489,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __pyx_r = __pyx_v_ngrams; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":74 + /* "gensim/models/_utils_any2vec.pyx":77 * * * cpdef compute_ngrams(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -2551,17 +2549,17 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams(PyObj case 1: if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_min_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 1); __PYX_ERR(0, 74, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 1); __PYX_ERR(0, 77, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 2); __PYX_ERR(0, 74, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 2); __PYX_ERR(0, 77, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams") < 0)) __PYX_ERR(0, 74, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams") < 0)) __PYX_ERR(0, 77, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { goto __pyx_L5_argtuple_error; @@ -2571,12 +2569,12 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams(PyObj values[2] = PyTuple_GET_ITEM(__pyx_args, 2); } __pyx_v_word = values[0]; - __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 74, __pyx_L3_error) - __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 74, __pyx_L3_error) + __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 77, __pyx_L3_error) + __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 77, __pyx_L3_error) } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 74, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 77, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -2595,7 +2593,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHO PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("compute_ngrams", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 74, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 77, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -5332,7 +5330,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {0, 0, 0, 0, 0, 0, 0} }; static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(0, 94, __pyx_L1_error) + __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(0, 97, __pyx_L1_error) __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 272, __pyx_L1_error) __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_n_s_RuntimeError); if (!__pyx_builtin_RuntimeError) __PYX_ERR(1, 856, __pyx_L1_error) __pyx_builtin_ImportError = __Pyx_GetBuiltinName(__pyx_n_s_ImportError); if (!__pyx_builtin_ImportError) __PYX_ERR(1, 1038, __pyx_L1_error) @@ -5751,62 +5749,62 @@ if (!__Pyx_RefNanny) { if (__Pyx_patch_abc() < 0) __PYX_ERR(0, 1, __pyx_L1_error) #endif - /* "gensim/models/_utils_any2vec.pyx":11 + /* "gensim/models/_utils_any2vec.pyx":14 + * ctypedef signed char int8_t * - * from libc.stdint cimport uint32_t, int8_t * from six import PY2 # <<<<<<<<<<<<<< * import numpy as np * cimport numpy as np */ - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 11, __pyx_L1_error) + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 14, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_n_s_PY2); __Pyx_GIVEREF(__pyx_n_s_PY2); PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_PY2); - __pyx_t_2 = __Pyx_Import(__pyx_n_s_six, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 11, __pyx_L1_error) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_six, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 14, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 11, __pyx_L1_error) + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 14, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_PY2, __pyx_t_1) < 0) __PYX_ERR(0, 11, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_PY2, __pyx_t_1) < 0) __PYX_ERR(0, 14, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "gensim/models/_utils_any2vec.pyx":12 - * from libc.stdint cimport uint32_t, int8_t + /* "gensim/models/_utils_any2vec.pyx":15 + * * from six import PY2 * import numpy as np # <<<<<<<<<<<<<< * cimport numpy as np * */ - __pyx_t_2 = __Pyx_Import(__pyx_n_s_numpy, 0, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 12, __pyx_L1_error) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_numpy, 0, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 15, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_np, __pyx_t_2) < 0) __PYX_ERR(0, 12, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_np, __pyx_t_2) < 0) __PYX_ERR(0, 15, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "gensim/models/_utils_any2vec.pyx":22 + /* "gensim/models/_utils_any2vec.pyx":25 * return ord(b) * * _byte_to_int = _byte_to_int_py2 if PY2 else _byte_to_int_py3 # <<<<<<<<<<<<<< * * */ - __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 22, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 25, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 22, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 25, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_3) { - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 22, __pyx_L1_error) + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 25, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __pyx_t_1; __pyx_t_1 = 0; } else { - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 22, __pyx_L1_error) + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 25, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __pyx_t_1; __pyx_t_1 = 0; } - if (PyDict_SetItem(__pyx_d, __pyx_n_s_byte_to_int, __pyx_t_2) < 0) __PYX_ERR(0, 22, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_byte_to_int, __pyx_t_2) < 0) __PYX_ERR(0, 25, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; /* "gensim/models/_utils_any2vec.pyx":1 diff --git a/gensim/models/_utils_any2vec.pyx b/gensim/models/_utils_any2vec.pyx index ce41eefb7a..22984026b1 100644 --- a/gensim/models/_utils_any2vec.pyx +++ b/gensim/models/_utils_any2vec.pyx @@ -7,7 +7,10 @@ """General functions used for any2vec models.""" -from libc.stdint cimport uint32_t, int8_t +cdef extern from "stdint_wrapper.h": + ctypedef unsigned int uint32_t + ctypedef signed char int8_t + from six import PY2 import numpy as np cimport numpy as np diff --git a/gensim/models/stdint_wrapper.h b/gensim/models/stdint_wrapper.h new file mode 100644 index 0000000000..e0e3b27c05 --- /dev/null +++ b/gensim/models/stdint_wrapper.h @@ -0,0 +1,17 @@ +/* + * This file is here to support older versions of the MSVC compiler that don't + * have stdint.h. + */ +#ifdef _MSC_VER + #ifndef _MSC_STDINT_H_ + #if _MSC_VER < 1300 + typedef unsigned char uint8_t; + typedef unsigned int uint32_t; + #else + typedef unsigned __int8 uint8_t; + typedef unsigned __int32 uint32_t; + #endif + #endif +#else + #include +#endif From fd3234085f0b39ab6956196b004cb2bc5936da9a Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Mon, 21 Jan 2019 19:23:53 +1100 Subject: [PATCH 08/32] fixup: add missing int8_t typedef --- gensim/models/stdint_wrapper.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gensim/models/stdint_wrapper.h b/gensim/models/stdint_wrapper.h index e0e3b27c05..0b9832dedf 100644 --- a/gensim/models/stdint_wrapper.h +++ b/gensim/models/stdint_wrapper.h @@ -7,9 +7,11 @@ #if _MSC_VER < 1300 typedef unsigned char uint8_t; typedef unsigned int uint32_t; + typedef char int8_t; #else typedef unsigned __int8 uint8_t; typedef unsigned __int32 uint32_t; + typedef char int8_t; #endif #endif #else From 8e7bd4007b40dd978489de397c4b80182abcdd90 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Tue, 22 Jan 2019 21:51:29 +1100 Subject: [PATCH 09/32] review response: avoid split and join --- gensim/models/_utils_any2vec.c | 1589 +++++++++++++++++++++--------- gensim/models/_utils_any2vec.pyx | 55 ++ gensim/models/fasttext.py | 12 +- gensim/models/keyedvectors.py | 27 +- gensim/models/utils_any2vec.py | 173 +++- 5 files changed, 1292 insertions(+), 564 deletions(-) diff --git a/gensim/models/_utils_any2vec.c b/gensim/models/_utils_any2vec.c index 67eee33698..04a0d2c2af 100644 --- a/gensim/models/_utils_any2vec.c +++ b/gensim/models/_utils_any2vec.c @@ -1286,45 +1286,49 @@ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ const char* function_name); -/* PyThreadStateGet.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; -#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; -#define __Pyx_PyErr_Occurred() __pyx_tstate->curexc_type -#else -#define __Pyx_PyThreadState_declare -#define __Pyx_PyThreadState_assign -#define __Pyx_PyErr_Occurred() PyErr_Occurred() -#endif +/* GetItemInt.proto */ +#define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Fast(o, (Py_ssize_t)i, is_list, wraparound, boundscheck) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL) :\ + __Pyx_GetItemInt_Generic(o, to_py_func(i)))) +#define __Pyx_GetItemInt_List(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_List_Fast(o, (Py_ssize_t)i, wraparound, boundscheck) :\ + (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck); +#define __Pyx_GetItemInt_Tuple(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Tuple_Fast(o, (Py_ssize_t)i, wraparound, boundscheck) :\ + (PyErr_SetString(PyExc_IndexError, "tuple index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck); +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j); +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, + int is_list, int wraparound, int boundscheck); -/* PyErrFetchRestore.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) -#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) -#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) -#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) -#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) -static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); -static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); -#if CYTHON_COMPILING_IN_CPYTHON -#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) -#else -#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) -#endif +/* GetModuleGlobalName.proto */ +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_d))) ?\ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} +#define __Pyx_GetModuleGlobalNameUncached(var, name) {\ + PY_UINT64_T __pyx_dict_version;\ + PyObject *__pyx_dict_cached_value;\ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); #else -#define __Pyx_PyErr_Clear() PyErr_Clear() -#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) -#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) -#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) -#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); #endif -/* RaiseException.proto */ -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); - /* PyCFunctionFastCall.proto */ #if CYTHON_FAST_PYCCALL static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); @@ -1363,6 +1367,45 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject /* PyObjectCallOneArg.proto */ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#define __Pyx_PyErr_Occurred() __pyx_tstate->curexc_type +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* RaiseException.proto */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + /* DictGetItem.proto */ #if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key); @@ -1499,27 +1542,6 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); /* ImportFrom.proto */ static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); -/* GetModuleGlobalName.proto */ -#if CYTHON_USE_DICT_VERSIONS -#define __Pyx_GetModuleGlobalName(var, name) {\ - static PY_UINT64_T __pyx_dict_version = 0;\ - static PyObject *__pyx_dict_cached_value = NULL;\ - (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_d))) ?\ - (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ - __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ -} -#define __Pyx_GetModuleGlobalNameUncached(var, name) {\ - PY_UINT64_T __pyx_dict_version;\ - PyObject *__pyx_dict_cached_value;\ - (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ -} -static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); -#else -#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) -#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) -static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); -#endif - /* CLineInTraceback.proto */ #ifdef CYTHON_CLINE_IN_TRACEBACK #define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) @@ -1555,6 +1577,9 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint32_t(uint32_t value); /* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value); +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); + /* RealImag.proto */ #if CYTHON_CCOMPLEX #ifdef __cplusplus @@ -1653,9 +1678,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value) #endif #endif -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); - /* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_enum__NPY_TYPES(enum NPY_TYPES value); @@ -1727,6 +1749,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObj static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject *, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObject *, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObject *, unsigned int, unsigned int, int __pyx_skip_dispatch); /*proto*/ +static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(PyObject *, unsigned int, unsigned int, int __pyx_skip_dispatch); /*proto*/ static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*)(PyObject *)); /*proto*/ #define __Pyx_MODULE_NAME "gensim.models._utils_any2vec" extern int __pyx_module_is_main_gensim__models___utils_any2vec; @@ -1753,6 +1776,8 @@ static const char __pyx_k_min_n[] = "min_n"; static const char __pyx_k_numpy[] = "numpy"; static const char __pyx_k_range[] = "range"; static const char __pyx_k_import[] = "__import__"; +static const char __pyx_k_MB_MASK[] = "_MB_MASK"; +static const char __pyx_k_MB_START[] = "_MB_START"; static const char __pyx_k_ValueError[] = "ValueError"; static const char __pyx_k_ImportError[] = "ImportError"; static const char __pyx_k_byte_to_int[] = "_byte_to_int"; @@ -1774,6 +1799,8 @@ static PyObject *__pyx_kp_u_; static PyObject *__pyx_kp_u_Format_string_allocated_too_shor; static PyObject *__pyx_kp_u_Format_string_allocated_too_shor_2; static PyObject *__pyx_n_s_ImportError; +static PyObject *__pyx_n_s_MB_MASK; +static PyObject *__pyx_n_s_MB_START; static PyObject *__pyx_kp_u_Non_native_byte_order_not_suppor; static PyObject *__pyx_n_s_PY2; static PyObject *__pyx_n_s_Pyx_CFunc_object____object___t; @@ -1805,12 +1832,15 @@ static PyObject *__pyx_n_s_wrap; static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_bytez); /* proto */ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string); /* proto */ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info); /* proto */ static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(PyObject *__pyx_self, PyObject *__pyx_v_b); /* proto */ static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_int_0; static PyObject *__pyx_int_1; +static PyObject *__pyx_int_128; +static PyObject *__pyx_int_192; static PyObject *__pyx_tuple__3; static PyObject *__pyx_tuple__4; static PyObject *__pyx_tuple__5; @@ -2433,6 +2463,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) # <<<<<<<<<<<<<< * return ngrams + * */ __Pyx_INCREF(__pyx_v_i); __pyx_t_1 = __pyx_v_i; @@ -2483,6 +2514,8 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) * return ngrams # <<<<<<<<<<<<<< + * + * # */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_ngrams); @@ -2610,115 +2643,579 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHO return __pyx_r; } -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 - * # experimental exception made for __getbuffer__ and __releasebuffer__ - * # -- the details of this may change. - * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< - * # This implementation of getbuffer is geared towards Cython - * # requirements, and does not yet fulfill the PEP. +/* "gensim/models/_utils_any2vec.pyx":110 + * + * + * cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< + * """Computes ngrams for a word. + * */ -/* Python wrapper */ -static CYTHON_UNUSED int __pyx_pw_5numpy_7ndarray_1__getbuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ -static CYTHON_UNUSED int __pyx_pw_5numpy_7ndarray_1__getbuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__getbuffer__ (wrapper)", 0); - __pyx_r = __pyx_pf_5numpy_7ndarray___getbuffer__(((PyArrayObject *)__pyx_v_self), ((Py_buffer *)__pyx_v_info), ((int)__pyx_v_flags)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n, CYTHON_UNUSED int __pyx_skip_dispatch) { + PyObject *__pyx_v_utf8_word = 0; + int __pyx_v_num_bytes; + int __pyx_v_j; int __pyx_v_i; - int __pyx_v_ndim; - int __pyx_v_endian_detector; - int __pyx_v_little_endian; - int __pyx_v_t; - char *__pyx_v_f; - PyArray_Descr *__pyx_v_descr = 0; - int __pyx_v_offset; - int __pyx_r; + int __pyx_v_n; + PyObject *__pyx_v_ngrams = NULL; + PyObject *__pyx_v_ngram = NULL; + PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations - int __pyx_t_1; - int __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - int __pyx_t_4; + PyObject *__pyx_t_1 = NULL; + Py_ssize_t __pyx_t_2; + Py_UCS4 __pyx_t_3; + PyObject *__pyx_t_4 = NULL; int __pyx_t_5; int __pyx_t_6; - PyArray_Descr *__pyx_t_7; + int __pyx_t_7; PyObject *__pyx_t_8 = NULL; - char *__pyx_t_9; - if (__pyx_v_info == NULL) { - PyErr_SetString(PyExc_BufferError, "PyObject_GetBuffer: view==NULL argument is obsolete"); - return -1; - } - __Pyx_RefNannySetupContext("__getbuffer__", 0); - __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); - __Pyx_GIVEREF(__pyx_v_info->obj); + int __pyx_t_9; + int __pyx_t_10; + int __pyx_t_11; + int __pyx_t_12; + int __pyx_t_13; + int __pyx_t_14; + __Pyx_RefNannySetupContext("compute_ngrams_bytes", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":265 + /* "gensim/models/_utils_any2vec.pyx":134 * - * cdef int i, ndim - * cdef int endian_detector = 1 # <<<<<<<<<<<<<< - * cdef bint little_endian = ((&endian_detector)[0] != 0) + * """ + * cdef bytes utf8_word = f'<{word}>'.encode("utf-8") # <<<<<<<<<<<<<< + * cdef int num_bytes = len(utf8_word) + * cdef int j, i, n + */ + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 134, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = 0; + __pyx_t_3 = 127; + __Pyx_INCREF(__pyx_kp_u_); + __pyx_t_2 += 1; + __Pyx_GIVEREF(__pyx_kp_u_); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_kp_u_); + __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_word, __pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 134, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_3 = (__Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) > __pyx_t_3) ? __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) : __pyx_t_3; + __pyx_t_2 += __Pyx_PyUnicode_GET_LENGTH(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_4); + __pyx_t_4 = 0; + __Pyx_INCREF(__pyx_kp_u__2); + __pyx_t_2 += 1; + __Pyx_GIVEREF(__pyx_kp_u__2); + PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_kp_u__2); + __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_1, 3, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 134, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyUnicode_AsUTF8String(((PyObject*)__pyx_t_4)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 134, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_utf8_word = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "gensim/models/_utils_any2vec.pyx":135 + * """ + * cdef bytes utf8_word = f'<{word}>'.encode("utf-8") + * cdef int num_bytes = len(utf8_word) # <<<<<<<<<<<<<< + * cdef int j, i, n * */ - __pyx_v_endian_detector = 1; + if (unlikely(__pyx_v_utf8_word == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 135, __pyx_L1_error) + } + __pyx_t_2 = PyBytes_GET_SIZE(__pyx_v_utf8_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 135, __pyx_L1_error) + __pyx_v_num_bytes = __pyx_t_2; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":266 - * cdef int i, ndim - * cdef int endian_detector = 1 - * cdef bint little_endian = ((&endian_detector)[0] != 0) # <<<<<<<<<<<<<< + /* "gensim/models/_utils_any2vec.pyx":138 + * cdef int j, i, n * - * ndim = PyArray_NDIM(self) + * ngrams = [] # <<<<<<<<<<<<<< + * for i in range(num_bytes): + * ngram = [] */ - __pyx_v_little_endian = ((((char *)(&__pyx_v_endian_detector))[0]) != 0); + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 138, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_ngrams = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":268 - * cdef bint little_endian = ((&endian_detector)[0] != 0) + /* "gensim/models/_utils_any2vec.pyx":139 * - * ndim = PyArray_NDIM(self) # <<<<<<<<<<<<<< + * ngrams = [] + * for i in range(num_bytes): # <<<<<<<<<<<<<< + * ngram = [] * - * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) */ - __pyx_v_ndim = PyArray_NDIM(__pyx_v_self); + __pyx_t_5 = __pyx_v_num_bytes; + __pyx_t_6 = __pyx_t_5; + for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_6; __pyx_t_7+=1) { + __pyx_v_i = __pyx_t_7; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 - * ndim = PyArray_NDIM(self) + /* "gensim/models/_utils_any2vec.pyx":140 + * ngrams = [] + * for i in range(num_bytes): + * ngram = [] # <<<<<<<<<<<<<< * - * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< - * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): - * raise ValueError(u"ndarray is not C contiguous") + * if utf8_word[i] & _MB_MASK == _MB_START: */ - __pyx_t_2 = (((__pyx_v_flags & PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS) != 0); - if (__pyx_t_2) { - } else { - __pyx_t_1 = __pyx_t_2; - goto __pyx_L4_bool_binop_done; - } + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 140, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_XDECREF_SET(__pyx_v_ngram, ((PyObject*)__pyx_t_1)); + __pyx_t_1 = 0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":271 + /* "gensim/models/_utils_any2vec.pyx":142 + * ngram = [] * - * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) - * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): # <<<<<<<<<<<<<< - * raise ValueError(u"ndarray is not C contiguous") + * if utf8_word[i] & _MB_MASK == _MB_START: # <<<<<<<<<<<<<< + * continue * */ - __pyx_t_2 = ((!(PyArray_CHKFLAGS(__pyx_v_self, NPY_ARRAY_C_CONTIGUOUS) != 0)) != 0); - __pyx_t_1 = __pyx_t_2; - __pyx_L4_bool_binop_done:; + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 142, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 142, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_8 = PyNumber_And(__pyx_t_1, __pyx_t_4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 142, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 142, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = PyObject_RichCompare(__pyx_t_8, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 142, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_9 < 0)) __PYX_ERR(0, 142, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_9) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 - * ndim = PyArray_NDIM(self) + /* "gensim/models/_utils_any2vec.pyx":143 * - * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< - * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): - * raise ValueError(u"ndarray is not C contiguous") + * if utf8_word[i] & _MB_MASK == _MB_START: + * continue # <<<<<<<<<<<<<< + * + * j, n = i, 1 */ - if (unlikely(__pyx_t_1)) { + goto __pyx_L3_continue; + + /* "gensim/models/_utils_any2vec.pyx":142 + * ngram = [] + * + * if utf8_word[i] & _MB_MASK == _MB_START: # <<<<<<<<<<<<<< + * continue + * + */ + } + + /* "gensim/models/_utils_any2vec.pyx":145 + * continue + * + * j, n = i, 1 # <<<<<<<<<<<<<< + * while j < num_bytes and n <= max_n: + * ngram.append(utf8_word[j]) + */ + __pyx_t_10 = __pyx_v_i; + __pyx_t_11 = 1; + __pyx_v_j = __pyx_t_10; + __pyx_v_n = __pyx_t_11; + + /* "gensim/models/_utils_any2vec.pyx":146 + * + * j, n = i, 1 + * while j < num_bytes and n <= max_n: # <<<<<<<<<<<<<< + * ngram.append(utf8_word[j]) + * j += 1 + */ + while (1) { + __pyx_t_12 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); + if (__pyx_t_12) { + } else { + __pyx_t_9 = __pyx_t_12; + goto __pyx_L8_bool_binop_done; + } + __pyx_t_12 = ((__pyx_v_n <= __pyx_v_max_n) != 0); + __pyx_t_9 = __pyx_t_12; + __pyx_L8_bool_binop_done:; + if (!__pyx_t_9) break; + + /* "gensim/models/_utils_any2vec.pyx":147 + * j, n = i, 1 + * while j < num_bytes and n <= max_n: + * ngram.append(utf8_word[j]) # <<<<<<<<<<<<<< + * j += 1 + * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: + */ + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 147, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_13 = __Pyx_PyList_Append(__pyx_v_ngram, __pyx_t_1); if (unlikely(__pyx_t_13 == ((int)-1))) __PYX_ERR(0, 147, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "gensim/models/_utils_any2vec.pyx":148 + * while j < num_bytes and n <= max_n: + * ngram.append(utf8_word[j]) + * j += 1 # <<<<<<<<<<<<<< + * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: + * ngram.append(utf8_word[j]) + */ + __pyx_v_j = (__pyx_v_j + 1); + + /* "gensim/models/_utils_any2vec.pyx":149 + * ngram.append(utf8_word[j]) + * j += 1 + * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: # <<<<<<<<<<<<<< + * ngram.append(utf8_word[j]) + * j += 1 + */ + while (1) { + __pyx_t_12 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); + if (__pyx_t_12) { + } else { + __pyx_t_9 = __pyx_t_12; + goto __pyx_L12_bool_binop_done; + } + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 149, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 149, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_8 = PyNumber_And(__pyx_t_1, __pyx_t_4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 149, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 149, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = PyObject_RichCompare(__pyx_t_8, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 149, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_12 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_12 < 0)) __PYX_ERR(0, 149, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_9 = __pyx_t_12; + __pyx_L12_bool_binop_done:; + if (!__pyx_t_9) break; + + /* "gensim/models/_utils_any2vec.pyx":150 + * j += 1 + * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: + * ngram.append(utf8_word[j]) # <<<<<<<<<<<<<< + * j += 1 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + */ + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 150, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_13 = __Pyx_PyList_Append(__pyx_v_ngram, __pyx_t_1); if (unlikely(__pyx_t_13 == ((int)-1))) __PYX_ERR(0, 150, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "gensim/models/_utils_any2vec.pyx":151 + * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: + * ngram.append(utf8_word[j]) + * j += 1 # <<<<<<<<<<<<<< + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + * ngrams.append(bytes(ngram)) + */ + __pyx_v_j = (__pyx_v_j + 1); + } + + /* "gensim/models/_utils_any2vec.pyx":152 + * ngram.append(utf8_word[j]) + * j += 1 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< + * ngrams.append(bytes(ngram)) + * n += 1 + */ + __pyx_t_12 = ((__pyx_v_n >= __pyx_v_min_n) != 0); + if (__pyx_t_12) { + } else { + __pyx_t_9 = __pyx_t_12; + goto __pyx_L15_bool_binop_done; + } + __pyx_t_14 = ((__pyx_v_n == 1) != 0); + if (__pyx_t_14) { + } else { + __pyx_t_12 = __pyx_t_14; + goto __pyx_L17_bool_binop_done; + } + __pyx_t_14 = ((__pyx_v_i == 0) != 0); + if (!__pyx_t_14) { + } else { + __pyx_t_12 = __pyx_t_14; + goto __pyx_L17_bool_binop_done; + } + __pyx_t_14 = ((__pyx_v_j == __pyx_v_num_bytes) != 0); + __pyx_t_12 = __pyx_t_14; + __pyx_L17_bool_binop_done:; + __pyx_t_14 = ((!__pyx_t_12) != 0); + __pyx_t_9 = __pyx_t_14; + __pyx_L15_bool_binop_done:; + if (__pyx_t_9) { + + /* "gensim/models/_utils_any2vec.pyx":153 + * j += 1 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + * ngrams.append(bytes(ngram)) # <<<<<<<<<<<<<< + * n += 1 + * return ngrams + */ + __pyx_t_1 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_v_ngram); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 153, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_13 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_1); if (unlikely(__pyx_t_13 == ((int)-1))) __PYX_ERR(0, 153, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "gensim/models/_utils_any2vec.pyx":152 + * ngram.append(utf8_word[j]) + * j += 1 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< + * ngrams.append(bytes(ngram)) + * n += 1 + */ + } + + /* "gensim/models/_utils_any2vec.pyx":154 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + * ngrams.append(bytes(ngram)) + * n += 1 # <<<<<<<<<<<<<< + * return ngrams + */ + __pyx_v_n = (__pyx_v_n + 1); + } + __pyx_L3_continue:; + } + + /* "gensim/models/_utils_any2vec.pyx":155 + * ngrams.append(bytes(ngram)) + * n += 1 + * return ngrams # <<<<<<<<<<<<<< + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_ngrams); + __pyx_r = __pyx_v_ngrams; + goto __pyx_L0; + + /* "gensim/models/_utils_any2vec.pyx":110 + * + * + * cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< + * """Computes ngrams for a word. + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_utf8_word); + __Pyx_XDECREF(__pyx_v_ngrams); + __Pyx_XDECREF(__pyx_v_ngram); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes[] = "compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n)\nComputes ngrams for a word.\n\n Ported from the original FB implementation.\n\n Parameters\n ----------\n word : str\n A unicode string.\n min_n : unsigned int\n The minimum ngram length.\n max_n : unsigned int\n The maximum ngram length.\n\n Returns:\n --------\n list\n A list of ngrams, where each ngram is a list of **bytes**.\n\n See Also\n --------\n `Original implementation `__\n\n "; +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_word = 0; + unsigned int __pyx_v_min_n; + unsigned int __pyx_v_max_n; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("compute_ngrams_bytes (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_word,&__pyx_n_s_min_n,&__pyx_n_s_max_n,0}; + PyObject* values[3] = {0,0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_word)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_min_n)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 1); __PYX_ERR(0, 110, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 2: + if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_n)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 2); __PYX_ERR(0, 110, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams_bytes") < 0)) __PYX_ERR(0, 110, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + } + __pyx_v_word = values[0]; + __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 110, __pyx_L3_error) + __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 110, __pyx_L3_error) + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 110, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes(__pyx_self, __pyx_v_word, __pyx_v_min_n, __pyx_v_max_n); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("compute_ngrams_bytes", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 110, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 + * # experimental exception made for __getbuffer__ and __releasebuffer__ + * # -- the details of this may change. + * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< + * # This implementation of getbuffer is geared towards Cython + * # requirements, and does not yet fulfill the PEP. + */ + +/* Python wrapper */ +static CYTHON_UNUSED int __pyx_pw_5numpy_7ndarray_1__getbuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +static CYTHON_UNUSED int __pyx_pw_5numpy_7ndarray_1__getbuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getbuffer__ (wrapper)", 0); + __pyx_r = __pyx_pf_5numpy_7ndarray___getbuffer__(((PyArrayObject *)__pyx_v_self), ((Py_buffer *)__pyx_v_info), ((int)__pyx_v_flags)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + int __pyx_v_i; + int __pyx_v_ndim; + int __pyx_v_endian_detector; + int __pyx_v_little_endian; + int __pyx_v_t; + char *__pyx_v_f; + PyArray_Descr *__pyx_v_descr = 0; + int __pyx_v_offset; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + int __pyx_t_5; + int __pyx_t_6; + PyArray_Descr *__pyx_t_7; + PyObject *__pyx_t_8 = NULL; + char *__pyx_t_9; + if (__pyx_v_info == NULL) { + PyErr_SetString(PyExc_BufferError, "PyObject_GetBuffer: view==NULL argument is obsolete"); + return -1; + } + __Pyx_RefNannySetupContext("__getbuffer__", 0); + __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(__pyx_v_info->obj); + + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":265 + * + * cdef int i, ndim + * cdef int endian_detector = 1 # <<<<<<<<<<<<<< + * cdef bint little_endian = ((&endian_detector)[0] != 0) + * + */ + __pyx_v_endian_detector = 1; + + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":266 + * cdef int i, ndim + * cdef int endian_detector = 1 + * cdef bint little_endian = ((&endian_detector)[0] != 0) # <<<<<<<<<<<<<< + * + * ndim = PyArray_NDIM(self) + */ + __pyx_v_little_endian = ((((char *)(&__pyx_v_endian_detector))[0]) != 0); + + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":268 + * cdef bint little_endian = ((&endian_detector)[0] != 0) + * + * ndim = PyArray_NDIM(self) # <<<<<<<<<<<<<< + * + * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) + */ + __pyx_v_ndim = PyArray_NDIM(__pyx_v_self); + + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + * ndim = PyArray_NDIM(self) + * + * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< + * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): + * raise ValueError(u"ndarray is not C contiguous") + */ + __pyx_t_2 = (((__pyx_v_flags & PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS) != 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":271 + * + * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) + * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): # <<<<<<<<<<<<<< + * raise ValueError(u"ndarray is not C contiguous") + * + */ + __pyx_t_2 = ((!(PyArray_CHKFLAGS(__pyx_v_self, NPY_ARRAY_C_CONTIGUOUS) != 0)) != 0); + __pyx_t_1 = __pyx_t_2; + __pyx_L4_bool_binop_done:; + + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + * ndim = PyArray_NDIM(self) + * + * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< + * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): + * raise ValueError(u"ndarray is not C contiguous") + */ + if (unlikely(__pyx_t_1)) { /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) @@ -5250,6 +5747,7 @@ static PyMethodDef __pyx_methods[] = { {"ft_hash_bytes", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash_bytes, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_ft_hash_bytes}, {"ft_hash_broken", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_2ft_hash_broken}, {"compute_ngrams", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams, METH_VARARGS|METH_KEYWORDS, __pyx_doc_6gensim_6models_14_utils_any2vec_4compute_ngrams}, + {"compute_ngrams_bytes", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes, METH_VARARGS|METH_KEYWORDS, __pyx_doc_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes}, {0, 0, 0, 0} }; @@ -5299,6 +5797,8 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_u_Format_string_allocated_too_shor, __pyx_k_Format_string_allocated_too_shor, sizeof(__pyx_k_Format_string_allocated_too_shor), 0, 1, 0, 0}, {&__pyx_kp_u_Format_string_allocated_too_shor_2, __pyx_k_Format_string_allocated_too_shor_2, sizeof(__pyx_k_Format_string_allocated_too_shor_2), 0, 1, 0, 0}, {&__pyx_n_s_ImportError, __pyx_k_ImportError, sizeof(__pyx_k_ImportError), 0, 0, 1, 1}, + {&__pyx_n_s_MB_MASK, __pyx_k_MB_MASK, sizeof(__pyx_k_MB_MASK), 0, 0, 1, 1}, + {&__pyx_n_s_MB_START, __pyx_k_MB_START, sizeof(__pyx_k_MB_START), 0, 0, 1, 1}, {&__pyx_kp_u_Non_native_byte_order_not_suppor, __pyx_k_Non_native_byte_order_not_suppor, sizeof(__pyx_k_Non_native_byte_order_not_suppor), 0, 1, 0, 0}, {&__pyx_n_s_PY2, __pyx_k_PY2, sizeof(__pyx_k_PY2), 0, 0, 1, 1}, {&__pyx_n_s_Pyx_CFunc_object____object___t, __pyx_k_Pyx_CFunc_object____object___t, sizeof(__pyx_k_Pyx_CFunc_object____object___t), 0, 0, 1, 1}, @@ -5442,6 +5942,8 @@ static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void) { if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(0, 1, __pyx_L1_error); __pyx_int_0 = PyInt_FromLong(0); if (unlikely(!__pyx_int_0)) __PYX_ERR(0, 1, __pyx_L1_error) __pyx_int_1 = PyInt_FromLong(1); if (unlikely(!__pyx_int_1)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_int_128 = PyInt_FromLong(128); if (unlikely(!__pyx_int_128)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_int_192 = PyInt_FromLong(192); if (unlikely(!__pyx_int_192)) __PYX_ERR(0, 1, __pyx_L1_error) return 0; __pyx_L1_error:; return -1; @@ -5807,6 +6309,24 @@ if (!__Pyx_RefNanny) { if (PyDict_SetItem(__pyx_d, __pyx_n_s_byte_to_int, __pyx_t_2) < 0) __PYX_ERR(0, 25, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + /* "gensim/models/_utils_any2vec.pyx":106 + * # as opposed to a new character. + * # + * _MB_MASK = 0xC0 # <<<<<<<<<<<<<< + * _MB_START = 0x80 + * + */ + if (PyDict_SetItem(__pyx_d, __pyx_n_s_MB_MASK, __pyx_int_192) < 0) __PYX_ERR(0, 106, __pyx_L1_error) + + /* "gensim/models/_utils_any2vec.pyx":107 + * # + * _MB_MASK = 0xC0 + * _MB_START = 0x80 # <<<<<<<<<<<<<< + * + * + */ + if (PyDict_SetItem(__pyx_d, __pyx_n_s_MB_START, __pyx_int_128) < 0) __PYX_ERR(0, 107, __pyx_L1_error) + /* "gensim/models/_utils_any2vec.pyx":1 * #!/usr/bin/env cython # <<<<<<<<<<<<<< * # cython: boundscheck=False @@ -6344,26 +6864,350 @@ static int __Pyx_ParseOptionalKeywords( goto invalid_keyword; } } - return 0; -arg_passed_twice: - __Pyx_RaiseDoubleKeywordsError(function_name, key); - goto bad; -invalid_keyword_type: - PyErr_Format(PyExc_TypeError, - "%.200s() keywords must be strings", function_name); - goto bad; -invalid_keyword: - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION < 3 - "%.200s() got an unexpected keyword argument '%.200s'", - function_name, PyString_AsString(key)); - #else - "%s() got an unexpected keyword argument '%U'", - function_name, key); - #endif -bad: - return -1; + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION < 3 + "%.200s() got an unexpected keyword argument '%.200s'", + function_name, PyString_AsString(key)); + #else + "%s() got an unexpected keyword argument '%U'", + function_name, key); + #endif +bad: + return -1; +} + +/* GetItemInt */ +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { + PyObject *r; + if (!j) return NULL; + r = PyObject_GetItem(o, j); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyList_GET_SIZE(o); + } + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyList_GET_SIZE(o)))) { + PyObject *r = PyList_GET_ITEM(o, wrapped_i); + Py_INCREF(r); + return r; + } + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +#else + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyTuple_GET_SIZE(o); + } + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyTuple_GET_SIZE(o)))) { + PyObject *r = PyTuple_GET_ITEM(o, wrapped_i); + Py_INCREF(r); + return r; + } + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +#else + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS && CYTHON_USE_TYPE_SLOTS + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); + if ((!boundscheck) || (likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o))))) { + PyObject *r = PyList_GET_ITEM(o, n); + Py_INCREF(r); + return r; + } + } + else if (PyTuple_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); + if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyTuple_GET_SIZE(o)))) { + PyObject *r = PyTuple_GET_ITEM(o, n); + Py_INCREF(r); + return r; + } + } else { + PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; + if (likely(m && m->sq_item)) { + if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { + Py_ssize_t l = m->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + PyErr_Clear(); + } + } + return m->sq_item(o, i); + } + } +#else + if (is_list || PySequence_Check(o)) { + return PySequence_GetItem(o, i); + } +#endif + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +} + +/* GetModuleGlobalName */ +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ + PyObject *result; +#if !CYTHON_AVOID_BORROWED_REFS +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 + result = _PyDict_GetItem_KnownHash(__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } else if (unlikely(PyErr_Occurred())) { + return NULL; + } +#else + result = PyDict_GetItem(__pyx_d, name); + __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } +#endif +#else + result = PyObject_GetItem(__pyx_d, name); + __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); +} + +/* PyCFunctionFastCall */ +#if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { + PyCFunctionObject *func = (PyCFunctionObject*)func_obj; + PyCFunction meth = PyCFunction_GET_FUNCTION(func); + PyObject *self = PyCFunction_GET_SELF(func); + int flags = PyCFunction_GET_FLAGS(func); + assert(PyCFunction_Check(func)); + assert(METH_FASTCALL == (flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))); + assert(nargs >= 0); + assert(nargs == 0 || args != NULL); + /* _PyCFunction_FastCallDict() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!PyErr_Occurred()); + if ((PY_VERSION_HEX < 0x030700A0) || unlikely(flags & METH_KEYWORDS)) { + return (*((__Pyx_PyCFunctionFastWithKeywords)(void*)meth)) (self, args, nargs, NULL); + } else { + return (*((__Pyx_PyCFunctionFast)(void*)meth)) (self, args, nargs); + } +} +#endif + +/* PyFunctionFastCall */ +#if CYTHON_FAST_PYCALL +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = __Pyx_PyFrame_GetLocalsplus(f); + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; +#if PY_MAJOR_VERSION >= 3 + PyObject *kwdefs; +#endif + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { + return NULL; + } + if ( +#if PY_MAJOR_VERSION >= 3 + co->co_kwonlyargcount == 0 && +#endif + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); +#if PY_MAJOR_VERSION >= 3 + kwdefs = PyFunction_GET_KW_DEFAULTS(func); +#endif + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } +#if PY_MAJOR_VERSION >= 3 + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); +#else + result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, closure); +#endif + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif +#endif + +/* PyObjectCallMethO */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = PyCFunction_GET_FUNCTION(func); + self = PyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallOneArg */ +#if CYTHON_COMPILING_IN_CPYTHON +static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_New(1); + if (unlikely(!args)) return NULL; + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 0, arg); + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, &arg, 1); + } +#endif + if (likely(PyCFunction_Check(func))) { + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, arg); +#if CYTHON_FAST_PYCCALL + } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { + return __Pyx_PyCFunction_FastCall(func, &arg, 1); +#endif + } + } + return __Pyx__PyObject_CallOneArg(func, arg); +} +#else +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_Pack(1, arg); + if (unlikely(!args)) return NULL; + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; } +#endif /* PyErrFetchRestore */ #if CYTHON_FAST_THREAD_STATE @@ -6474,279 +7318,77 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject } else { type = instance_class; } - } - } - if (!instance_class) { - PyObject *args; - if (!value) - args = PyTuple_New(0); - else if (PyTuple_Check(value)) { - Py_INCREF(value); - args = value; - } else - args = PyTuple_Pack(1, value); - if (!args) - goto bad; - owned_instance = PyObject_Call(type, args, NULL); - Py_DECREF(args); - if (!owned_instance) - goto bad; - value = owned_instance; - if (!PyExceptionInstance_Check(value)) { - PyErr_Format(PyExc_TypeError, - "calling %R should have returned an instance of " - "BaseException, not %R", - type, Py_TYPE(value)); - goto bad; - } - } - } else { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto bad; - } - if (cause) { - PyObject *fixed_cause; - if (cause == Py_None) { - fixed_cause = NULL; - } else if (PyExceptionClass_Check(cause)) { - fixed_cause = PyObject_CallObject(cause, NULL); - if (fixed_cause == NULL) - goto bad; - } else if (PyExceptionInstance_Check(cause)) { - fixed_cause = cause; - Py_INCREF(fixed_cause); - } else { - PyErr_SetString(PyExc_TypeError, - "exception causes must derive from " - "BaseException"); - goto bad; - } - PyException_SetCause(value, fixed_cause); - } - PyErr_SetObject(type, value); - if (tb) { -#if CYTHON_COMPILING_IN_PYPY - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); - Py_INCREF(tb); - PyErr_Restore(tmp_type, tmp_value, tb); - Py_XDECREF(tmp_tb); -#else - PyThreadState *tstate = __Pyx_PyThreadState_Current; - PyObject* tmp_tb = tstate->curexc_traceback; - if (tb != tmp_tb) { - Py_INCREF(tb); - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_tb); - } -#endif - } -bad: - Py_XDECREF(owned_instance); - return; -} -#endif - -/* PyCFunctionFastCall */ -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { - PyCFunctionObject *func = (PyCFunctionObject*)func_obj; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - int flags = PyCFunction_GET_FLAGS(func); - assert(PyCFunction_Check(func)); - assert(METH_FASTCALL == (flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))); - assert(nargs >= 0); - assert(nargs == 0 || args != NULL); - /* _PyCFunction_FastCallDict() must not be called with an exception set, - because it may clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!PyErr_Occurred()); - if ((PY_VERSION_HEX < 0x030700A0) || unlikely(flags & METH_KEYWORDS)) { - return (*((__Pyx_PyCFunctionFastWithKeywords)(void*)meth)) (self, args, nargs, NULL); - } else { - return (*((__Pyx_PyCFunctionFast)(void*)meth)) (self, args, nargs); - } -} -#endif - -/* PyFunctionFastCall */ -#if CYTHON_FAST_PYCALL -static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, - PyObject *globals) { - PyFrameObject *f; - PyThreadState *tstate = __Pyx_PyThreadState_Current; - PyObject **fastlocals; - Py_ssize_t i; - PyObject *result; - assert(globals != NULL); - /* XXX Perhaps we should create a specialized - PyFrame_New() that doesn't take locals, but does - take builtins without sanity checking them. - */ - assert(tstate != NULL); - f = PyFrame_New(tstate, co, globals, NULL); - if (f == NULL) { - return NULL; - } - fastlocals = __Pyx_PyFrame_GetLocalsplus(f); - for (i = 0; i < na; i++) { - Py_INCREF(*args); - fastlocals[i] = *args++; - } - result = PyEval_EvalFrameEx(f,0); - ++tstate->recursion_depth; - Py_DECREF(f); - --tstate->recursion_depth; - return result; -} -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { - PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); - PyObject *globals = PyFunction_GET_GLOBALS(func); - PyObject *argdefs = PyFunction_GET_DEFAULTS(func); - PyObject *closure; -#if PY_MAJOR_VERSION >= 3 - PyObject *kwdefs; -#endif - PyObject *kwtuple, **k; - PyObject **d; - Py_ssize_t nd; - Py_ssize_t nk; - PyObject *result; - assert(kwargs == NULL || PyDict_Check(kwargs)); - nk = kwargs ? PyDict_Size(kwargs) : 0; - if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { - return NULL; - } - if ( -#if PY_MAJOR_VERSION >= 3 - co->co_kwonlyargcount == 0 && -#endif - likely(kwargs == NULL || nk == 0) && - co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { - if (argdefs == NULL && co->co_argcount == nargs) { - result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); - goto done; - } - else if (nargs == 0 && argdefs != NULL - && co->co_argcount == Py_SIZE(argdefs)) { - /* function called with no arguments, but all parameters have - a default value: use default values as arguments .*/ - args = &PyTuple_GET_ITEM(argdefs, 0); - result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); - goto done; - } - } - if (kwargs != NULL) { - Py_ssize_t pos, i; - kwtuple = PyTuple_New(2 * nk); - if (kwtuple == NULL) { - result = NULL; - goto done; + } } - k = &PyTuple_GET_ITEM(kwtuple, 0); - pos = i = 0; - while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { - Py_INCREF(k[i]); - Py_INCREF(k[i+1]); - i += 2; + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } } - nk = i / 2; - } - else { - kwtuple = NULL; - k = NULL; - } - closure = PyFunction_GET_CLOSURE(func); -#if PY_MAJOR_VERSION >= 3 - kwdefs = PyFunction_GET_KW_DEFAULTS(func); -#endif - if (argdefs != NULL) { - d = &PyTuple_GET_ITEM(argdefs, 0); - nd = Py_SIZE(argdefs); + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; } - else { - d = NULL; - nd = 0; + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); } -#if PY_MAJOR_VERSION >= 3 - result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, kwdefs, closure); + PyErr_SetObject(type, value); + if (tb) { +#if CYTHON_COMPILING_IN_PYPY + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); #else - result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, closure); -#endif - Py_XDECREF(kwtuple); -done: - Py_LeaveRecursiveCall(); - return result; -} -#endif -#endif - -/* PyObjectCallMethO */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { - PyObject *self, *result; - PyCFunction cfunc; - cfunc = PyCFunction_GET_FUNCTION(func); - self = PyCFunction_GET_SELF(func); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = cfunc(self, arg); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); - } - return result; -} -#endif - -/* PyObjectCallOneArg */ -#if CYTHON_COMPILING_IN_CPYTHON -static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_New(1); - if (unlikely(!args)) return NULL; - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 0, arg); - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, &arg, 1); - } -#endif - if (likely(PyCFunction_Check(func))) { - if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { - return __Pyx_PyObject_CallMethO(func, arg); -#if CYTHON_FAST_PYCCALL - } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { - return __Pyx_PyCFunction_FastCall(func, &arg, 1); -#endif + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); } +#endif } - return __Pyx__PyObject_CallOneArg(func, arg); -} -#else -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_Pack(1, arg); - if (unlikely(!args)) return NULL; - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; +bad: + Py_XDECREF(owned_instance); + return; } #endif @@ -7775,41 +8417,6 @@ static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { return value; } -/* GetModuleGlobalName */ -#if CYTHON_USE_DICT_VERSIONS -static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) -#else -static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) -#endif -{ - PyObject *result; -#if !CYTHON_AVOID_BORROWED_REFS -#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 - result = _PyDict_GetItem_KnownHash(__pyx_d, name, ((PyASCIIObject *) name)->hash); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } else if (unlikely(PyErr_Occurred())) { - return NULL; - } -#else - result = PyDict_GetItem(__pyx_d, name); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } -#endif -#else - result = PyObject_GetItem(__pyx_d, name); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } - PyErr_Clear(); -#endif - return __Pyx_GetBuiltinName(name); -} - /* CLineInTraceback */ #ifndef CYTHON_CLINE_IN_TRACEBACK static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { @@ -8132,6 +8739,37 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value) } } +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { + const int neg_one = (int) ((int) 0 - (int) 1), const_zero = (int) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(int) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(int) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(int), + little, !is_unsigned); + } +} + /* Declarations */ #if CYTHON_CCOMPLEX #ifdef __cplusplus @@ -8442,37 +9080,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value) #endif #endif -/* CIntToPy */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { - const int neg_one = (int) ((int) 0 - (int) 1), const_zero = (int) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(int) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(int) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(int) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(int), - little, !is_unsigned); - } -} - /* CIntToPy */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_enum__NPY_TYPES(enum NPY_TYPES value) { const enum NPY_TYPES neg_one = (enum NPY_TYPES) ((enum NPY_TYPES) 0 - (enum NPY_TYPES) 1), const_zero = (enum NPY_TYPES) 0; diff --git a/gensim/models/_utils_any2vec.pyx b/gensim/models/_utils_any2vec.pyx index 22984026b1..d398e21244 100644 --- a/gensim/models/_utils_any2vec.pyx +++ b/gensim/models/_utils_any2vec.pyx @@ -98,3 +98,58 @@ cpdef compute_ngrams(word, unsigned int min_n, unsigned int max_n): for i in range(0, len(extended_word) - ngram_length + 1): ngrams.append(extended_word[i:i + ngram_length]) return ngrams + +# +# UTF-8 bytes that begin with 10 are subsequent bytes of a multi-byte sequence, +# as opposed to a new character. +# +_MB_MASK = 0xC0 +_MB_START = 0x80 + + +cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): + """Computes ngrams for a word. + + Ported from the original FB implementation. + + Parameters + ---------- + word : str + A unicode string. + min_n : unsigned int + The minimum ngram length. + max_n : unsigned int + The maximum ngram length. + + Returns: + -------- + list + A list of ngrams, where each ngram is a list of **bytes**. + + See Also + -------- + `Original implementation `__ + + """ + cdef bytes utf8_word = f'<{word}>'.encode("utf-8") + cdef int num_bytes = len(utf8_word) + cdef int j, i, n + + ngrams = [] + for i in range(num_bytes): + ngram = [] + + if utf8_word[i] & _MB_MASK == _MB_START: + continue + + j, n = i, 1 + while j < num_bytes and n <= max_n: + ngram.append(utf8_word[j]) + j += 1 + while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: + ngram.append(utf8_word[j]) + j += 1 + if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + ngrams.append(bytes(ngram)) + n += 1 + return ngrams diff --git a/gensim/models/fasttext.py b/gensim/models/fasttext.py index a07d84fbf4..b42460dccc 100644 --- a/gensim/models/fasttext.py +++ b/gensim/models/fasttext.py @@ -98,7 +98,7 @@ from gensim.models.word2vec import Word2VecVocab, Word2VecTrainables, train_sg_pair, train_cbow_pair from gensim.models.keyedvectors import FastTextKeyedVectors from gensim.models.base_any2vec import BaseWordEmbeddingsModel -from gensim.models.utils_any2vec import ft_ngram_hashes, ft_ngram_hashes_broken +from gensim.models.utils_any2vec import ft_ngram_hashes from smart_open import smart_open from gensim.utils import deprecated, call_on_class_only @@ -554,8 +554,6 @@ def _clear_post_train(self): self.wv.buckets_word = None def estimate_memory(self, vocab_size=None, report=None): - hash_fn = ft_ngram_hashes if self.wv.compatible_hash else ft_ngram_hashes_broken - vocab_size = vocab_size or len(self.wv.vocab) vec_size = self.vector_size * np.dtype(np.float32).itemsize l1_size = self.trainables.layer1_size * np.dtype(np.float32).itemsize @@ -574,7 +572,13 @@ def estimate_memory(self, vocab_size=None, report=None): buckets = set() num_ngrams = 0 for word in self.wv.vocab: - hashes = hash_fn(word, self.wv.min_n, self.wv.max_n, self.trainables.bucket) + hashes = ft_ngram_hashes( + word, + self.wv.min_n, + self.wv.max_n, + self.trainables.bucket, + self.wv.compatible_hash + ) num_ngrams += len(hashes) buckets.update(hashes) num_buckets = len(buckets) diff --git a/gensim/models/keyedvectors.py b/gensim/models/keyedvectors.py index 26d6ecbf98..cbbbeaad45 100644 --- a/gensim/models/keyedvectors.py +++ b/gensim/models/keyedvectors.py @@ -183,7 +183,6 @@ _save_word2vec_format, _load_word2vec_format, ft_ngram_hashes, - ft_ngram_hashes_broken, ) from gensim.similarities.termsim import TermSimilarityIndex, SparseTermSimilarityMatrix @@ -2011,8 +2010,7 @@ def __contains__(self, word): if word in self.vocab: return True else: - hash_fn = ft_ngram_hashes if self.compatible_hash else ft_ngram_hashes_broken - hashes = hash_fn(word, self.min_n, self.max_n, self.bucket) + hashes = ft_ngram_hashes(word, self.min_n, self.max_n, self.bucket, self.compatible_hash) return any(h in self.hash2index for h in hashes) def save(self, *args, **kwargs): @@ -2055,8 +2053,6 @@ def word_vec(self, word, use_norm=False): If word and all ngrams not in vocabulary. """ - hash_fn = ft_ngram_hashes if self.compatible_hash else ft_ngram_hashes_broken - if word in self.vocab: return super(FastTextKeyedVectors, self).word_vec(word, use_norm) elif self.bucket == 0: @@ -2068,7 +2064,7 @@ def word_vec(self, word, use_norm=False): else: ngram_weights = self.vectors_ngrams ngrams_found = 0 - for ngram_hash in hash_fn(word, self.min_n, self.max_n, self.bucket): + for ngram_hash in ft_ngram_hashes(word, self.min_n, self.max_n, self.bucket, self.compatible_hash): if ngram_hash in self.hash2index: word_vec += ngram_weights[self.hash2index[ngram_hash]] ngrams_found += 1 @@ -2126,7 +2122,7 @@ def init_ngrams_weights(self, seed): self.min_n, self.max_n, self.bucket, - ft_ngram_hashes if self.compatible_hash else ft_ngram_hashes_broken, + self.compatible_hash, self.hash2index ) self.num_ngram_vectors = len(ngram_indices) @@ -2149,7 +2145,7 @@ def update_ngrams_weights(self, seed, old_vocab_len): self.min_n, self.max_n, self.bucket, - ft_ngram_hashes if self.compatible_hash else ft_ngram_hashes_broken, + self.compatible_hash, self.hash2index ) num_new_ngrams = len(new_ngram_hashes) @@ -2211,7 +2207,7 @@ def init_post_load(self, vectors, match_gensim=False): self.min_n, self.max_n, self.bucket, - ft_ngram_hashes if self.compatible_hash else ft_ngram_hashes_broken, + self.compatible_hash, dict(), # we don't care what goes here in this case ) ngram_hashes = sorted(set(ngram_hashes)) @@ -2233,18 +2229,16 @@ def adjust_vectors(self): if self.bucket == 0: return - hash_fn = ft_ngram_hashes if self.compatible_hash else ft_ngram_hashes_broken - for w, v in self.vocab.items(): word_vec = np.copy(self.vectors_vocab[v.index]) - ngram_hashes = hash_fn(w, self.min_n, self.max_n, self.bucket) + ngram_hashes = ft_ngram_hashes(w, self.min_n, self.max_n, self.bucket, self.compatible_hash) for nh in ngram_hashes: word_vec += self.vectors_ngrams[self.hash2index[nh]] word_vec /= len(ngram_hashes) + 1 self.vectors[v.index] = word_vec -def _process_fasttext_vocab(iterable, min_n, max_n, num_buckets, hash_fn, hash2index): +def _process_fasttext_vocab(iterable, min_n, max_n, num_buckets, compatible_hash, hash2index): """ Performs a common operation for FastText weight initialization and updates: scan the vocabulary, calculate ngrams and their hashes, keep @@ -2261,8 +2255,9 @@ def _process_fasttext_vocab(iterable, min_n, max_n, num_buckets, hash_fn, hash2i The maximum length of ngrams. num_buckets : int The number of buckets used by the model. - hash_fn : callable - Used to hash ngrams to buckets. + compatible_hash : boolean + True for compatibility with the Facebook implementation. + False for compatibility with the old Gensim implementation. hash2index : dict Updated in-place. @@ -2288,7 +2283,7 @@ def _process_fasttext_vocab(iterable, min_n, max_n, num_buckets, hash_fn, hash2i for word, vocab in iterable: wi = [] - for ngram_hash in hash_fn(word, min_n, max_n, num_buckets): + for ngram_hash in ft_ngram_hashes(word, min_n, max_n, num_buckets, compatible_hash): if ngram_hash not in hash2index: # # This is a new ngram. Reserve a new index in hash2index. diff --git a/gensim/models/utils_any2vec.py b/gensim/models/utils_any2vec.py index 7b6e9a499a..9083fa8064 100644 --- a/gensim/models/utils_any2vec.py +++ b/gensim/models/utils_any2vec.py @@ -10,13 +10,18 @@ extensions for FastText. If they are not available, then the module substitutes slower Python versions in their place. -Another related set of FastText functionality is computing ngrams for a word, -and then hashing them. The :py:func:`ft_ngram_hashes` and -:py:func:`ft_ngram_hashes_broken` achieves this goal. +Another related set of FastText functionality is computing ngrams for a word. +The :py:func:`compute_ngrams` and :py:func:`compute_ngrams_bytes` hashes achieve that. -Finally, the module also exposes "working" and "broken" hash functions for -FastText. It does this without abstracting them away, because the correct -function to use depends on the current model. +Closely related is the functionality for hashing ngrams, implemented by the +:py:func:`ft_hash` and :py:func:`ft_hash_broken` functions. +The module exposes "working" and "broken" hash functions in order to maintain +backwards compatibility with older versions of Gensim. + +For compatibility with older Gensim, use :py:func:`compute_ngrams` and +:py:func:`ft_hash_broken` to has each ngram. For compatibility with the +current Facebook implementation, use :py:func:`compute_ngrams_bytes` and +:py:func:`ft_hash_bytes`. """ @@ -32,6 +37,14 @@ logger = logging.getLogger(__name__) +# +# UTF-8 bytes that begin with 10 are subsequent bytes of a multi-byte sequence, +# as opposed to a new character. +# +_MB_MASK = 0xC0 +_MB_START = 0x80 + + def _byte_to_int_py3(b): return b @@ -47,7 +60,7 @@ def _byte_to_int_py2(b): # Define this here so we can unittest this function directly. # Only use this function if the faster C version fails to import. # -def _ft_hash_py_bytes(bytez): +def _ft_hash_bytes_py(bytez): """Calculate hash based on `bytez`. Reproduce `hash method from Facebook fastText implementation `_. @@ -72,7 +85,7 @@ def _ft_hash_py_bytes(bytez): return h -def _ft_hash_py_broken(string): +def _ft_hash_broken_py(string): """Calculate hash based on `string`. Reproduce `hash method from Facebook fastText implementation `_. @@ -100,62 +113,109 @@ def _ft_hash_py_broken(string): return h -try: - from gensim.models._utils_any2vec import ( - compute_ngrams, - ft_hash_broken as _ft_hash_cy_broken, - ft_hash_bytes as _ft_hash_cy_bytes, - ) - _ft_hash_bytes = _ft_hash_cy_bytes - _ft_hash_broken = _ft_hash_cy_broken - FAST_VERSION = 0 -except ImportError: - # failed... fall back to plain python - FAST_VERSION = -1 - _ft_hash_bytes = _ft_hash_py_bytes - _ft_hash_broken = _ft_hash_py_broken +def _compute_ngrams_py(word, min_n, max_n): + """Get the list of all possible ngrams for a given word. + Parameters + ---------- + word : str + The word whose ngrams need to be computed. + min_n : int + Minimum character length of the ngrams. + max_n : int + Maximum character length of the ngrams. + Returns + ------- + list of str + Sequence of character ngrams. + """ + BOW, EOW = ('<', '>') # Used by FastText to attach to all words as prefix and suffix + extended_word = BOW + word + EOW + ngrams = [] + for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): + for i in range(0, len(extended_word) - ngram_length + 1): + ngrams.append(extended_word[i:i + ngram_length]) + return ngrams -def ft_ngram_hashes(word, minn, maxn, num_buckets): - """Calculate the ngrams of the word and hash them. +def _compute_ngrams_bytes_py(word, min_n, max_n): + """Computes ngrams for a word. - Do this in a way that is compatible with the original Facebook implementation. + Ported from the original FB implementation. Parameters ---------- word : str - The word to calculate ngram hashes for. - minn : int - Minimum ngram length - maxn : int - Maximum ngram length - num_buckets : int - The number of buckets + A unicode string. + min_n : unsigned int + The minimum ngram length. + max_n : unsigned int + The maximum ngram length. - Returns - ------- - A list of hashes (integers), one per each detected ngram. + Returns: + -------- + list of str + A list of ngrams, where each ngram is a list of **bytes**. + + See Also + -------- + `Original implementation `__ # noqa: E501 """ - ngrams = compute_ngrams(word, minn, maxn) - # - # This is a trick to avoid encoding each ngram separately, which is - # computationally expensive. It works for two reasons: - # - # 1) ngrams are guaranteed to _not_ have any spaces, because that is the - # character we use to split sentences into words. - # 2) space is an ASCII character, so it survives UTF-8 encoding unchanged. - # - encoded_ngrams = (" ".join(ngrams)).encode("utf-8").split() - hashes = [_ft_hash_bytes(en) % num_buckets for en in encoded_ngrams] - return hashes + utf8_word = f'<{word}>'.encode("utf-8") + num_bytes = len(utf8_word) + n = 0 + + ngrams = [] + for i in range(num_bytes): + ngram = [] + + if utf8_word[i] & _MB_MASK == _MB_START: + continue + + j, n = i, 1 + while j < num_bytes and n <= max_n: + ngram.append(utf8_word[j]) + j += 1 + while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: + ngram.append(utf8_word[j]) + j += 1 + if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + ngrams.append(bytes(ngram)) + n += 1 + return ngrams -def ft_ngram_hashes_broken(word, minn, maxn, num_buckets): - """Calculate the ngrams of the word and hash them. +# +# Internally, we use the following convention to abstract away the presence +# or absence of the Cython extensions: +# +# - _function_cy: Imported from Cython extension +# - _function_py: Implemented in Python +# - function: Exported by this module. +# +try: + from gensim.models._utils_any2vec import ( + compute_ngrams as _compute_ngrams_cy, + compute_ngrams_bytes as _compute_ngrams_bytes_cy, + ft_hash_broken as _ft_hash_broken_cy, + ft_hash_bytes as _ft_hash_bytes_cy, + ) + ft_hash_bytes = _ft_hash_bytes_cy + ft_hash_broken = _ft_hash_broken_cy + compute_ngrams = _compute_ngrams_cy + compute_ngrams_bytes = _compute_ngrams_bytes_cy + FAST_VERSION = 0 +except ImportError: + # failed... fall back to plain python + FAST_VERSION = -1 + ft_hash_bytes = _ft_hash_bytes_py + ft_hash_broken = _ft_hash_broken_py + compute_ngrams = _compute_ngrams_py + compute_ngrams_bytes = _compute_ngrams_bytes_py - Do this in a way that is incompatible with the original Facebook - implementation, but compatible with older versions of Gensim. + +def ft_ngram_hashes(word, minn, maxn, num_buckets, fb_compatible=True): + """Calculate the ngrams of the word and hash them. Parameters ---------- @@ -167,14 +227,21 @@ def ft_ngram_hashes_broken(word, minn, maxn, num_buckets): Maximum ngram length num_buckets : int The number of buckets + fb_compatible : boolean, optional + True for compatibility with the Facebook implementation. + False for compatibility with the old Gensim implementation. Returns ------- A list of hashes (integers), one per each detected ngram. """ - ngrams = compute_ngrams(word, minn, maxn) - hashes = [_ft_hash_broken(n) % num_buckets for n in ngrams] + if fb_compatible: + encoded_ngrams = compute_ngrams_bytes(word, minn, maxn) + hashes = [ft_hash_bytes(n) % num_buckets for n in encoded_ngrams] + else: + text_ngrams = compute_ngrams(word, minn, maxn) + hashes = [ft_hash_broken(n) % num_buckets for n in text_ngrams] return hashes From 2ea46721f6d3d8e5871f4b35141686554fe2056c Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Tue, 22 Jan 2019 21:52:33 +1100 Subject: [PATCH 10/32] review response: add comment to explain hack --- gensim/models/_utils_any2vec.c | 318 +++++++++++++++---------------- gensim/models/_utils_any2vec.pyx | 5 +- 2 files changed, 163 insertions(+), 160 deletions(-) diff --git a/gensim/models/_utils_any2vec.c b/gensim/models/_utils_any2vec.c index 04a0d2c2af..e66d4bd506 100644 --- a/gensim/models/_utils_any2vec.c +++ b/gensim/models/_utils_any2vec.c @@ -1852,7 +1852,7 @@ static PyObject *__pyx_tuple__10; static PyObject *__pyx_codeobj__11; /* Late includes */ -/* "gensim/models/_utils_any2vec.pyx":19 +/* "gensim/models/_utils_any2vec.pyx":22 * * * cdef _byte_to_int_py3(b): # <<<<<<<<<<<<<< @@ -1865,7 +1865,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3(PyObj __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("_byte_to_int_py3", 0); - /* "gensim/models/_utils_any2vec.pyx":20 + /* "gensim/models/_utils_any2vec.pyx":23 * * cdef _byte_to_int_py3(b): * return b # <<<<<<<<<<<<<< @@ -1877,7 +1877,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3(PyObj __pyx_r = __pyx_v_b; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":19 + /* "gensim/models/_utils_any2vec.pyx":22 * * * cdef _byte_to_int_py3(b): # <<<<<<<<<<<<<< @@ -1892,7 +1892,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3(PyObj return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":22 +/* "gensim/models/_utils_any2vec.pyx":25 * return b * * cdef _byte_to_int_py2(b): # <<<<<<<<<<<<<< @@ -1907,7 +1907,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObj PyObject *__pyx_t_2 = NULL; __Pyx_RefNannySetupContext("_byte_to_int_py2", 0); - /* "gensim/models/_utils_any2vec.pyx":23 + /* "gensim/models/_utils_any2vec.pyx":26 * * cdef _byte_to_int_py2(b): * return ord(b) # <<<<<<<<<<<<<< @@ -1915,14 +1915,14 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObj * _byte_to_int = _byte_to_int_py2 if PY2 else _byte_to_int_py3 */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_Ord(__pyx_v_b); if (unlikely(__pyx_t_1 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 23, __pyx_L1_error) - __pyx_t_2 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 23, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Ord(__pyx_v_b); if (unlikely(__pyx_t_1 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 26, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 26, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":22 + /* "gensim/models/_utils_any2vec.pyx":25 * return b * * cdef _byte_to_int_py2(b): # <<<<<<<<<<<<<< @@ -1941,7 +1941,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObj return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":28 +/* "gensim/models/_utils_any2vec.pyx":31 * * * cpdef ft_hash_bytes(bytes bytez): # <<<<<<<<<<<<<< @@ -1963,7 +1963,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject PyObject *__pyx_t_6 = NULL; __Pyx_RefNannySetupContext("ft_hash_bytes", 0); - /* "gensim/models/_utils_any2vec.pyx":44 + /* "gensim/models/_utils_any2vec.pyx":47 * * """ * cdef uint32_t h = 2166136261 # <<<<<<<<<<<<<< @@ -1972,7 +1972,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject */ __pyx_v_h = 0x811C9DC5; - /* "gensim/models/_utils_any2vec.pyx":47 + /* "gensim/models/_utils_any2vec.pyx":50 * cdef char b * * for b in bytez: # <<<<<<<<<<<<<< @@ -1981,7 +1981,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject */ if (unlikely(__pyx_v_bytez == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); - __PYX_ERR(0, 47, __pyx_L1_error) + __PYX_ERR(0, 50, __pyx_L1_error) } __Pyx_INCREF(__pyx_v_bytez); __pyx_t_1 = __pyx_v_bytez; @@ -1991,7 +1991,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject __pyx_t_2 = __pyx_t_5; __pyx_v_b = (__pyx_t_2[0]); - /* "gensim/models/_utils_any2vec.pyx":48 + /* "gensim/models/_utils_any2vec.pyx":51 * * for b in bytez: * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss # <<<<<<<<<<<<<< @@ -2000,7 +2000,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject */ __pyx_v_h = (__pyx_v_h ^ ((uint32_t)((int8_t)__pyx_v_b))); - /* "gensim/models/_utils_any2vec.pyx":49 + /* "gensim/models/_utils_any2vec.pyx":52 * for b in bytez: * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss * h = h * 16777619 # <<<<<<<<<<<<<< @@ -2011,7 +2011,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":50 + /* "gensim/models/_utils_any2vec.pyx":53 * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss * h = h * 16777619 * return h # <<<<<<<<<<<<<< @@ -2019,13 +2019,13 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_6 = __Pyx_PyInt_From_uint32_t(__pyx_v_h); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 50, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyInt_From_uint32_t(__pyx_v_h); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 53, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_r = __pyx_t_6; __pyx_t_6 = 0; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":28 + /* "gensim/models/_utils_any2vec.pyx":31 * * * cpdef ft_hash_bytes(bytes bytez): # <<<<<<<<<<<<<< @@ -2052,7 +2052,7 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash_bytes(PyObje PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("ft_hash_bytes (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_bytez), (&PyBytes_Type), 1, "bytez", 1))) __PYX_ERR(0, 28, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_bytez), (&PyBytes_Type), 1, "bytez", 1))) __PYX_ERR(0, 31, __pyx_L1_error) __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(__pyx_self, ((PyObject*)__pyx_v_bytez)); /* function exit code */ @@ -2070,7 +2070,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(CYTHON_ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("ft_hash_bytes", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(__pyx_v_bytez, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 28, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(__pyx_v_bytez, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 31, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -2087,7 +2087,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(CYTHON_ return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":53 +/* "gensim/models/_utils_any2vec.pyx":56 * * * cpdef ft_hash_broken(unicode string): # <<<<<<<<<<<<<< @@ -2112,7 +2112,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec long __pyx_t_9; __Pyx_RefNannySetupContext("ft_hash_broken", 0); - /* "gensim/models/_utils_any2vec.pyx":70 + /* "gensim/models/_utils_any2vec.pyx":73 * * """ * cdef unsigned int h = 2166136261 # <<<<<<<<<<<<<< @@ -2121,7 +2121,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec */ __pyx_v_h = 0x811C9DC5; - /* "gensim/models/_utils_any2vec.pyx":71 + /* "gensim/models/_utils_any2vec.pyx":74 * """ * cdef unsigned int h = 2166136261 * for c in string: # <<<<<<<<<<<<<< @@ -2130,29 +2130,29 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec */ if (unlikely(__pyx_v_string == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); - __PYX_ERR(0, 71, __pyx_L1_error) + __PYX_ERR(0, 74, __pyx_L1_error) } __Pyx_INCREF(__pyx_v_string); __pyx_t_1 = __pyx_v_string; - __pyx_t_6 = __Pyx_init_unicode_iteration(__pyx_t_1, (&__pyx_t_3), (&__pyx_t_4), (&__pyx_t_5)); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 71, __pyx_L1_error) + __pyx_t_6 = __Pyx_init_unicode_iteration(__pyx_t_1, (&__pyx_t_3), (&__pyx_t_4), (&__pyx_t_5)); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 74, __pyx_L1_error) for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_3; __pyx_t_7++) { __pyx_t_2 = __pyx_t_7; - __pyx_t_8 = PyUnicode_FromOrdinal(__Pyx_PyUnicode_READ(__pyx_t_5, __pyx_t_4, __pyx_t_2)); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 71, __pyx_L1_error) + __pyx_t_8 = PyUnicode_FromOrdinal(__Pyx_PyUnicode_READ(__pyx_t_5, __pyx_t_4, __pyx_t_2)); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 74, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_XDECREF_SET(__pyx_v_c, __pyx_t_8); __pyx_t_8 = 0; - /* "gensim/models/_utils_any2vec.pyx":72 + /* "gensim/models/_utils_any2vec.pyx":75 * cdef unsigned int h = 2166136261 * for c in string: * h ^= ord(c) # <<<<<<<<<<<<<< * h *= 16777619 * return h */ - __pyx_t_9 = __Pyx_PyObject_Ord(__pyx_v_c); if (unlikely(__pyx_t_9 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 72, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyObject_Ord(__pyx_v_c); if (unlikely(__pyx_t_9 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 75, __pyx_L1_error) __pyx_v_h = (__pyx_v_h ^ __pyx_t_9); - /* "gensim/models/_utils_any2vec.pyx":73 + /* "gensim/models/_utils_any2vec.pyx":76 * for c in string: * h ^= ord(c) * h *= 16777619 # <<<<<<<<<<<<<< @@ -2163,7 +2163,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":74 + /* "gensim/models/_utils_any2vec.pyx":77 * h ^= ord(c) * h *= 16777619 * return h # <<<<<<<<<<<<<< @@ -2171,13 +2171,13 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_8 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 74, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 77, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __pyx_r = __pyx_t_8; __pyx_t_8 = 0; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":53 + /* "gensim/models/_utils_any2vec.pyx":56 * * * cpdef ft_hash_broken(unicode string): # <<<<<<<<<<<<<< @@ -2205,7 +2205,7 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken(PyObj PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("ft_hash_broken (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_string), (&PyUnicode_Type), 1, "string", 1))) __PYX_ERR(0, 53, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_string), (&PyUnicode_Type), 1, "string", 1))) __PYX_ERR(0, 56, __pyx_L1_error) __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(__pyx_self, ((PyObject*)__pyx_v_string)); /* function exit code */ @@ -2223,7 +2223,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHO PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("ft_hash_broken", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(__pyx_v_string, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 53, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(__pyx_v_string, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 56, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -2240,7 +2240,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHO return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":77 +/* "gensim/models/_utils_any2vec.pyx":80 * * * cpdef compute_ngrams(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -2272,14 +2272,14 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec int __pyx_t_14; __Pyx_RefNannySetupContext("compute_ngrams", 0); - /* "gensim/models/_utils_any2vec.pyx":95 + /* "gensim/models/_utils_any2vec.pyx":98 * * """ * cdef unicode extended_word = f'<{word}>' # <<<<<<<<<<<<<< * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): */ - __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 95, __pyx_L1_error) + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 98, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = 0; __pyx_t_3 = 127; @@ -2287,7 +2287,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __pyx_t_2 += 1; __Pyx_GIVEREF(__pyx_kp_u_); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_kp_u_); - __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_word, __pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 95, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_word, __pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 98, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = (__Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) > __pyx_t_3) ? __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) : __pyx_t_3; __pyx_t_2 += __Pyx_PyUnicode_GET_LENGTH(__pyx_t_4); @@ -2298,43 +2298,43 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __pyx_t_2 += 1; __Pyx_GIVEREF(__pyx_kp_u__2); PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_kp_u__2); - __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_1, 3, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 95, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_1, 3, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 98, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_extended_word = ((PyObject*)__pyx_t_4); __pyx_t_4 = 0; - /* "gensim/models/_utils_any2vec.pyx":96 + /* "gensim/models/_utils_any2vec.pyx":99 * """ * cdef unicode extended_word = f'<{word}>' * ngrams = [] # <<<<<<<<<<<<<< * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): */ - __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 96, __pyx_L1_error) + __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 99, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_v_ngrams = ((PyObject*)__pyx_t_4); __pyx_t_4 = 0; - /* "gensim/models/_utils_any2vec.pyx":97 + /* "gensim/models/_utils_any2vec.pyx":100 * cdef unicode extended_word = f'<{word}>' * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): # <<<<<<<<<<<<<< * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) */ - __pyx_t_4 = __Pyx_PyInt_From_unsigned_int(__pyx_v_min_n); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyInt_From_unsigned_int(__pyx_v_min_n); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 100, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = __pyx_v_max_n; - __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 100, __pyx_L1_error) if (((__pyx_t_5 < __pyx_t_2) != 0)) { __pyx_t_6 = __pyx_t_5; } else { __pyx_t_6 = __pyx_t_2; } - __pyx_t_1 = PyInt_FromSsize_t((__pyx_t_6 + 1)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_t_1 = PyInt_FromSsize_t((__pyx_t_6 + 1)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 100, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 100, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_4); @@ -2342,16 +2342,16 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_1); __pyx_t_4 = 0; __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 100, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (likely(PyList_CheckExact(__pyx_t_1)) || PyTuple_CheckExact(__pyx_t_1)) { __pyx_t_7 = __pyx_t_1; __Pyx_INCREF(__pyx_t_7); __pyx_t_6 = 0; __pyx_t_8 = NULL; } else { - __pyx_t_6 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_t_6 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 100, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_8 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_t_8 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 100, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; for (;;) { @@ -2359,17 +2359,17 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec if (likely(PyList_CheckExact(__pyx_t_7))) { if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_7)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_t_1 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 100, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 100, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } else { if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_7)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 100, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 100, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } @@ -2379,7 +2379,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(0, 97, __pyx_L1_error) + else __PYX_ERR(0, 100, __pyx_L1_error) } break; } @@ -2388,23 +2388,23 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __Pyx_XDECREF_SET(__pyx_v_ngram_length, __pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":98 + /* "gensim/models/_utils_any2vec.pyx":101 * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): # <<<<<<<<<<<<<< * ngrams.append(extended_word[i:i + ngram_length]) * return ngrams */ - __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 98, __pyx_L1_error) - __pyx_t_1 = PyInt_FromSsize_t(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 101, __pyx_L1_error) + __pyx_t_1 = PyInt_FromSsize_t(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 101, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_4 = PyNumber_Subtract(__pyx_t_1, __pyx_v_ngram_length); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_t_4 = PyNumber_Subtract(__pyx_t_1, __pyx_v_ngram_length); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 101, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyInt_AddObjC(__pyx_t_4, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_AddObjC(__pyx_t_4, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 101, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 101, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_INCREF(__pyx_int_0); __Pyx_GIVEREF(__pyx_int_0); @@ -2412,16 +2412,16 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 101, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (likely(PyList_CheckExact(__pyx_t_1)) || PyTuple_CheckExact(__pyx_t_1)) { __pyx_t_4 = __pyx_t_1; __Pyx_INCREF(__pyx_t_4); __pyx_t_2 = 0; __pyx_t_9 = NULL; } else { - __pyx_t_2 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_t_2 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 101, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_9 = Py_TYPE(__pyx_t_4)->tp_iternext; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_t_9 = Py_TYPE(__pyx_t_4)->tp_iternext; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 101, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; for (;;) { @@ -2429,17 +2429,17 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec if (likely(PyList_CheckExact(__pyx_t_4))) { if (__pyx_t_2 >= PyList_GET_SIZE(__pyx_t_4)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_t_1 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 101, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 101, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } else { if (__pyx_t_2 >= PyTuple_GET_SIZE(__pyx_t_4)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 101, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 101, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } @@ -2449,7 +2449,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(0, 98, __pyx_L1_error) + else __PYX_ERR(0, 101, __pyx_L1_error) } break; } @@ -2458,7 +2458,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __Pyx_XDECREF_SET(__pyx_v_i, __pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":99 + /* "gensim/models/_utils_any2vec.pyx":102 * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) # <<<<<<<<<<<<<< @@ -2471,26 +2471,26 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec if (__pyx_t_11) { __pyx_t_10 = 0; } else { - __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 99, __pyx_L1_error) + __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 102, __pyx_L1_error) __pyx_t_10 = __pyx_t_12; } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyNumber_Add(__pyx_v_i, __pyx_v_ngram_length); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 99, __pyx_L1_error) + __pyx_t_1 = PyNumber_Add(__pyx_v_i, __pyx_v_ngram_length); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 102, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_11 = (__pyx_t_1 == Py_None); if (__pyx_t_11) { __pyx_t_12 = PY_SSIZE_T_MAX; } else { - __pyx_t_13 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_13 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 99, __pyx_L1_error) + __pyx_t_13 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_13 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 102, __pyx_L1_error) __pyx_t_12 = __pyx_t_13; } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyUnicode_Substring(__pyx_v_extended_word, __pyx_t_10, __pyx_t_12); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 99, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyUnicode_Substring(__pyx_v_extended_word, __pyx_t_10, __pyx_t_12); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 102, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_1); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 99, __pyx_L1_error) + __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_1); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 102, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":98 + /* "gensim/models/_utils_any2vec.pyx":101 * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): # <<<<<<<<<<<<<< @@ -2500,7 +2500,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec } __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "gensim/models/_utils_any2vec.pyx":97 + /* "gensim/models/_utils_any2vec.pyx":100 * cdef unicode extended_word = f'<{word}>' * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): # <<<<<<<<<<<<<< @@ -2510,7 +2510,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - /* "gensim/models/_utils_any2vec.pyx":100 + /* "gensim/models/_utils_any2vec.pyx":103 * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) * return ngrams # <<<<<<<<<<<<<< @@ -2522,7 +2522,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __pyx_r = __pyx_v_ngrams; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":77 + /* "gensim/models/_utils_any2vec.pyx":80 * * * cpdef compute_ngrams(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -2582,17 +2582,17 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams(PyObj case 1: if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_min_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 1); __PYX_ERR(0, 77, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 1); __PYX_ERR(0, 80, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 2); __PYX_ERR(0, 77, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 2); __PYX_ERR(0, 80, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams") < 0)) __PYX_ERR(0, 77, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams") < 0)) __PYX_ERR(0, 80, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { goto __pyx_L5_argtuple_error; @@ -2602,12 +2602,12 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams(PyObj values[2] = PyTuple_GET_ITEM(__pyx_args, 2); } __pyx_v_word = values[0]; - __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 77, __pyx_L3_error) - __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 77, __pyx_L3_error) + __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 80, __pyx_L3_error) + __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 80, __pyx_L3_error) } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 77, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 80, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -2626,7 +2626,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHO PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("compute_ngrams", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 77, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 80, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -2643,7 +2643,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHO return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":110 +/* "gensim/models/_utils_any2vec.pyx":113 * * * cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -2678,14 +2678,14 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P int __pyx_t_14; __Pyx_RefNannySetupContext("compute_ngrams_bytes", 0); - /* "gensim/models/_utils_any2vec.pyx":134 + /* "gensim/models/_utils_any2vec.pyx":137 * * """ * cdef bytes utf8_word = f'<{word}>'.encode("utf-8") # <<<<<<<<<<<<<< * cdef int num_bytes = len(utf8_word) * cdef int j, i, n */ - __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 134, __pyx_L1_error) + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 137, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = 0; __pyx_t_3 = 127; @@ -2693,7 +2693,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_t_2 += 1; __Pyx_GIVEREF(__pyx_kp_u_); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_kp_u_); - __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_word, __pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 134, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_word, __pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 137, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = (__Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) > __pyx_t_3) ? __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) : __pyx_t_3; __pyx_t_2 += __Pyx_PyUnicode_GET_LENGTH(__pyx_t_4); @@ -2704,16 +2704,16 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_t_2 += 1; __Pyx_GIVEREF(__pyx_kp_u__2); PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_kp_u__2); - __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_1, 3, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 134, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_1, 3, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 137, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyUnicode_AsUTF8String(((PyObject*)__pyx_t_4)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 134, __pyx_L1_error) + __pyx_t_1 = PyUnicode_AsUTF8String(((PyObject*)__pyx_t_4)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 137, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_v_utf8_word = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":135 + /* "gensim/models/_utils_any2vec.pyx":138 * """ * cdef bytes utf8_word = f'<{word}>'.encode("utf-8") * cdef int num_bytes = len(utf8_word) # <<<<<<<<<<<<<< @@ -2722,24 +2722,24 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P */ if (unlikely(__pyx_v_utf8_word == Py_None)) { PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); - __PYX_ERR(0, 135, __pyx_L1_error) + __PYX_ERR(0, 138, __pyx_L1_error) } - __pyx_t_2 = PyBytes_GET_SIZE(__pyx_v_utf8_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 135, __pyx_L1_error) + __pyx_t_2 = PyBytes_GET_SIZE(__pyx_v_utf8_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 138, __pyx_L1_error) __pyx_v_num_bytes = __pyx_t_2; - /* "gensim/models/_utils_any2vec.pyx":138 + /* "gensim/models/_utils_any2vec.pyx":141 * cdef int j, i, n * * ngrams = [] # <<<<<<<<<<<<<< * for i in range(num_bytes): * ngram = [] */ - __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 138, __pyx_L1_error) + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_ngrams = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":139 + /* "gensim/models/_utils_any2vec.pyx":142 * * ngrams = [] * for i in range(num_bytes): # <<<<<<<<<<<<<< @@ -2751,43 +2751,43 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_6; __pyx_t_7+=1) { __pyx_v_i = __pyx_t_7; - /* "gensim/models/_utils_any2vec.pyx":140 + /* "gensim/models/_utils_any2vec.pyx":143 * ngrams = [] * for i in range(num_bytes): * ngram = [] # <<<<<<<<<<<<<< * * if utf8_word[i] & _MB_MASK == _MB_START: */ - __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 140, __pyx_L1_error) + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 143, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_XDECREF_SET(__pyx_v_ngram, ((PyObject*)__pyx_t_1)); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":142 + /* "gensim/models/_utils_any2vec.pyx":145 * ngram = [] * * if utf8_word[i] & _MB_MASK == _MB_START: # <<<<<<<<<<<<<< * continue * */ - __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 142, __pyx_L1_error) + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 142, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_8 = PyNumber_And(__pyx_t_1, __pyx_t_4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 142, __pyx_L1_error) + __pyx_t_8 = PyNumber_And(__pyx_t_1, __pyx_t_4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 142, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_1 = PyObject_RichCompare(__pyx_t_8, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 142, __pyx_L1_error) + __pyx_t_1 = PyObject_RichCompare(__pyx_t_8, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 145, __pyx_L1_error) __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_9 < 0)) __PYX_ERR(0, 142, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_9 < 0)) __PYX_ERR(0, 145, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_9) { - /* "gensim/models/_utils_any2vec.pyx":143 + /* "gensim/models/_utils_any2vec.pyx":146 * * if utf8_word[i] & _MB_MASK == _MB_START: * continue # <<<<<<<<<<<<<< @@ -2796,7 +2796,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P */ goto __pyx_L3_continue; - /* "gensim/models/_utils_any2vec.pyx":142 + /* "gensim/models/_utils_any2vec.pyx":145 * ngram = [] * * if utf8_word[i] & _MB_MASK == _MB_START: # <<<<<<<<<<<<<< @@ -2805,7 +2805,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P */ } - /* "gensim/models/_utils_any2vec.pyx":145 + /* "gensim/models/_utils_any2vec.pyx":148 * continue * * j, n = i, 1 # <<<<<<<<<<<<<< @@ -2817,7 +2817,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_v_j = __pyx_t_10; __pyx_v_n = __pyx_t_11; - /* "gensim/models/_utils_any2vec.pyx":146 + /* "gensim/models/_utils_any2vec.pyx":149 * * j, n = i, 1 * while j < num_bytes and n <= max_n: # <<<<<<<<<<<<<< @@ -2836,19 +2836,19 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_L8_bool_binop_done:; if (!__pyx_t_9) break; - /* "gensim/models/_utils_any2vec.pyx":147 + /* "gensim/models/_utils_any2vec.pyx":150 * j, n = i, 1 * while j < num_bytes and n <= max_n: * ngram.append(utf8_word[j]) # <<<<<<<<<<<<<< * j += 1 * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: */ - __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 147, __pyx_L1_error) + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 150, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_13 = __Pyx_PyList_Append(__pyx_v_ngram, __pyx_t_1); if (unlikely(__pyx_t_13 == ((int)-1))) __PYX_ERR(0, 147, __pyx_L1_error) + __pyx_t_13 = __Pyx_PyList_Append(__pyx_v_ngram, __pyx_t_1); if (unlikely(__pyx_t_13 == ((int)-1))) __PYX_ERR(0, 150, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":148 + /* "gensim/models/_utils_any2vec.pyx":151 * while j < num_bytes and n <= max_n: * ngram.append(utf8_word[j]) * j += 1 # <<<<<<<<<<<<<< @@ -2857,7 +2857,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P */ __pyx_v_j = (__pyx_v_j + 1); - /* "gensim/models/_utils_any2vec.pyx":149 + /* "gensim/models/_utils_any2vec.pyx":152 * ngram.append(utf8_word[j]) * j += 1 * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: # <<<<<<<<<<<<<< @@ -2871,38 +2871,38 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_t_9 = __pyx_t_12; goto __pyx_L12_bool_binop_done; } - __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 149, __pyx_L1_error) + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 152, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 149, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 152, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_8 = PyNumber_And(__pyx_t_1, __pyx_t_4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 149, __pyx_L1_error) + __pyx_t_8 = PyNumber_And(__pyx_t_1, __pyx_t_4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 152, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 149, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 152, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_1 = PyObject_RichCompare(__pyx_t_8, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 149, __pyx_L1_error) + __pyx_t_1 = PyObject_RichCompare(__pyx_t_8, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 152, __pyx_L1_error) __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_12 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_12 < 0)) __PYX_ERR(0, 149, __pyx_L1_error) + __pyx_t_12 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_12 < 0)) __PYX_ERR(0, 152, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_9 = __pyx_t_12; __pyx_L12_bool_binop_done:; if (!__pyx_t_9) break; - /* "gensim/models/_utils_any2vec.pyx":150 + /* "gensim/models/_utils_any2vec.pyx":153 * j += 1 * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: * ngram.append(utf8_word[j]) # <<<<<<<<<<<<<< * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): */ - __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 150, __pyx_L1_error) + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 153, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_13 = __Pyx_PyList_Append(__pyx_v_ngram, __pyx_t_1); if (unlikely(__pyx_t_13 == ((int)-1))) __PYX_ERR(0, 150, __pyx_L1_error) + __pyx_t_13 = __Pyx_PyList_Append(__pyx_v_ngram, __pyx_t_1); if (unlikely(__pyx_t_13 == ((int)-1))) __PYX_ERR(0, 153, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":151 + /* "gensim/models/_utils_any2vec.pyx":154 * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: * ngram.append(utf8_word[j]) * j += 1 # <<<<<<<<<<<<<< @@ -2912,7 +2912,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_v_j = (__pyx_v_j + 1); } - /* "gensim/models/_utils_any2vec.pyx":152 + /* "gensim/models/_utils_any2vec.pyx":155 * ngram.append(utf8_word[j]) * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< @@ -2945,19 +2945,19 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_L15_bool_binop_done:; if (__pyx_t_9) { - /* "gensim/models/_utils_any2vec.pyx":153 + /* "gensim/models/_utils_any2vec.pyx":156 * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): * ngrams.append(bytes(ngram)) # <<<<<<<<<<<<<< * n += 1 * return ngrams */ - __pyx_t_1 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_v_ngram); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 153, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_v_ngram); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 156, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_13 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_1); if (unlikely(__pyx_t_13 == ((int)-1))) __PYX_ERR(0, 153, __pyx_L1_error) + __pyx_t_13 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_1); if (unlikely(__pyx_t_13 == ((int)-1))) __PYX_ERR(0, 156, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":152 + /* "gensim/models/_utils_any2vec.pyx":155 * ngram.append(utf8_word[j]) * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< @@ -2966,7 +2966,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P */ } - /* "gensim/models/_utils_any2vec.pyx":154 + /* "gensim/models/_utils_any2vec.pyx":157 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): * ngrams.append(bytes(ngram)) * n += 1 # <<<<<<<<<<<<<< @@ -2977,7 +2977,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_L3_continue:; } - /* "gensim/models/_utils_any2vec.pyx":155 + /* "gensim/models/_utils_any2vec.pyx":158 * ngrams.append(bytes(ngram)) * n += 1 * return ngrams # <<<<<<<<<<<<<< @@ -2987,7 +2987,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_r = __pyx_v_ngrams; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":110 + /* "gensim/models/_utils_any2vec.pyx":113 * * * cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -3013,7 +3013,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P /* Python wrapper */ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes[] = "compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n)\nComputes ngrams for a word.\n\n Ported from the original FB implementation.\n\n Parameters\n ----------\n word : str\n A unicode string.\n min_n : unsigned int\n The minimum ngram length.\n max_n : unsigned int\n The maximum ngram length.\n\n Returns:\n --------\n list\n A list of ngrams, where each ngram is a list of **bytes**.\n\n See Also\n --------\n `Original implementation `__\n\n "; +static char __pyx_doc_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes[] = "compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n)\nComputes ngrams for a word.\n\n Ported from the original FB implementation.\n\n Parameters\n ----------\n word : str\n A unicode string.\n min_n : unsigned int\n The minimum ngram length.\n max_n : unsigned int\n The maximum ngram length.\n\n Returns:\n --------\n list of str\n A list of ngrams, where each ngram is a list of **bytes**.\n\n See Also\n --------\n `Original implementation `__\n\n "; static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_word = 0; unsigned int __pyx_v_min_n; @@ -3046,17 +3046,17 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes case 1: if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_min_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 1); __PYX_ERR(0, 110, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 1); __PYX_ERR(0, 113, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 2); __PYX_ERR(0, 110, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 2); __PYX_ERR(0, 113, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams_bytes") < 0)) __PYX_ERR(0, 110, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams_bytes") < 0)) __PYX_ERR(0, 113, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { goto __pyx_L5_argtuple_error; @@ -3066,12 +3066,12 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes values[2] = PyTuple_GET_ITEM(__pyx_args, 2); } __pyx_v_word = values[0]; - __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 110, __pyx_L3_error) - __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 110, __pyx_L3_error) + __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 113, __pyx_L3_error) + __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 113, __pyx_L3_error) } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 110, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 113, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -3090,7 +3090,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("compute_ngrams_bytes", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 110, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 113, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -5830,7 +5830,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {0, 0, 0, 0, 0, 0, 0} }; static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(0, 100, __pyx_L1_error) __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 272, __pyx_L1_error) __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_n_s_RuntimeError); if (!__pyx_builtin_RuntimeError) __PYX_ERR(1, 856, __pyx_L1_error) __pyx_builtin_ImportError = __Pyx_GetBuiltinName(__pyx_n_s_ImportError); if (!__pyx_builtin_ImportError) __PYX_ERR(1, 1038, __pyx_L1_error) @@ -6251,81 +6251,81 @@ if (!__Pyx_RefNanny) { if (__Pyx_patch_abc() < 0) __PYX_ERR(0, 1, __pyx_L1_error) #endif - /* "gensim/models/_utils_any2vec.pyx":14 + /* "gensim/models/_utils_any2vec.pyx":17 * ctypedef signed char int8_t * * from six import PY2 # <<<<<<<<<<<<<< * import numpy as np * cimport numpy as np */ - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 14, __pyx_L1_error) + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 17, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_n_s_PY2); __Pyx_GIVEREF(__pyx_n_s_PY2); PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_PY2); - __pyx_t_2 = __Pyx_Import(__pyx_n_s_six, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 14, __pyx_L1_error) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_six, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 17, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 14, __pyx_L1_error) + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 17, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_PY2, __pyx_t_1) < 0) __PYX_ERR(0, 14, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_PY2, __pyx_t_1) < 0) __PYX_ERR(0, 17, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "gensim/models/_utils_any2vec.pyx":15 + /* "gensim/models/_utils_any2vec.pyx":18 * * from six import PY2 * import numpy as np # <<<<<<<<<<<<<< * cimport numpy as np * */ - __pyx_t_2 = __Pyx_Import(__pyx_n_s_numpy, 0, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 15, __pyx_L1_error) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_numpy, 0, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 18, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_np, __pyx_t_2) < 0) __PYX_ERR(0, 15, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_np, __pyx_t_2) < 0) __PYX_ERR(0, 18, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "gensim/models/_utils_any2vec.pyx":25 + /* "gensim/models/_utils_any2vec.pyx":28 * return ord(b) * * _byte_to_int = _byte_to_int_py2 if PY2 else _byte_to_int_py3 # <<<<<<<<<<<<<< * * */ - __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 25, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 28, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 25, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 28, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_3) { - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 25, __pyx_L1_error) + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 28, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __pyx_t_1; __pyx_t_1 = 0; } else { - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 25, __pyx_L1_error) + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 28, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __pyx_t_1; __pyx_t_1 = 0; } - if (PyDict_SetItem(__pyx_d, __pyx_n_s_byte_to_int, __pyx_t_2) < 0) __PYX_ERR(0, 25, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_byte_to_int, __pyx_t_2) < 0) __PYX_ERR(0, 28, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "gensim/models/_utils_any2vec.pyx":106 + /* "gensim/models/_utils_any2vec.pyx":109 * # as opposed to a new character. * # * _MB_MASK = 0xC0 # <<<<<<<<<<<<<< * _MB_START = 0x80 * */ - if (PyDict_SetItem(__pyx_d, __pyx_n_s_MB_MASK, __pyx_int_192) < 0) __PYX_ERR(0, 106, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_MB_MASK, __pyx_int_192) < 0) __PYX_ERR(0, 109, __pyx_L1_error) - /* "gensim/models/_utils_any2vec.pyx":107 + /* "gensim/models/_utils_any2vec.pyx":110 * # * _MB_MASK = 0xC0 * _MB_START = 0x80 # <<<<<<<<<<<<<< * * */ - if (PyDict_SetItem(__pyx_d, __pyx_n_s_MB_START, __pyx_int_128) < 0) __PYX_ERR(0, 107, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_MB_START, __pyx_int_128) < 0) __PYX_ERR(0, 110, __pyx_L1_error) /* "gensim/models/_utils_any2vec.pyx":1 * #!/usr/bin/env cython # <<<<<<<<<<<<<< diff --git a/gensim/models/_utils_any2vec.pyx b/gensim/models/_utils_any2vec.pyx index d398e21244..187b1905bd 100644 --- a/gensim/models/_utils_any2vec.pyx +++ b/gensim/models/_utils_any2vec.pyx @@ -7,6 +7,9 @@ """General functions used for any2vec models.""" +# +# This is here to support older versions of the MSVC compiler that don't have stdint.h. +# cdef extern from "stdint_wrapper.h": ctypedef unsigned int uint32_t ctypedef signed char int8_t @@ -123,7 +126,7 @@ cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): Returns: -------- - list + list of str A list of ngrams, where each ngram is a list of **bytes**. See Also From 9137a27b67ac726fc32800beaa3414619e53452c Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Tue, 22 Jan 2019 21:53:46 +1100 Subject: [PATCH 11/32] review response: improve logging message --- gensim/models/fasttext.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gensim/models/fasttext.py b/gensim/models/fasttext.py index b42460dccc..34ddbdb428 100644 --- a/gensim/models/fasttext.py +++ b/gensim/models/fasttext.py @@ -712,7 +712,8 @@ def train(self, sentences=None, corpus_file=None, total_examples=None, total_wor if cant_train: raise ValueError( 'this model cannot be trained any further, ' - 'if this is a native model, try loading it with full_model=True' + 'if this is a native model, try loading it with ' + 'with FastText.load_fasttext_format(path, full_model=True)' ) super(FastText, self).train( From a654c3ca99d7a132fb7e3b925c97d5e594d95d06 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Tue, 22 Jan 2019 21:56:59 +1100 Subject: [PATCH 12/32] review response: fix hash_main function --- gensim/test/test_utils.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/gensim/test/test_utils.py b/gensim/test/test_utils.py index 0be21b94ca..27fde0e20d 100644 --- a/gensim/test/test_utils.py +++ b/gensim/test/test_utils.py @@ -269,14 +269,13 @@ def hash_main(alg): import six assert six.PY3, 'this only works under Py3' - assert gensim.models.utils_any2vec.FAST_VERSION == 0, 'Cython extensions are broken' + assert not DISABLE_CYTHON_TESTS, 'this only works if Cython extensions available' hashmap = { - 'py_broken': gensim.models.utils_any2vec._ft_hash_py_broken, - 'py_bytes': gensim.models.utils_any2vec._ft_hash_py_bytes, - 'cy': gensim.models.utils_any2vec._ft_hash_cy, - 'cy_broken': gensim.models.utils_any2vec._ft_hash_py_broken, - 'cy_bytes': gensim.models.utils_any2vec._ft_hash_cy_bytes, + 'py_broken': gensim.models.utils_any2vec._ft_hash_broken_py, + 'py_bytes': gensim.models.utils_any2vec._ft_hash_bytes_py, + 'cy_broken': gensim.models.utils_any2vec._ft_hash_broken_py, + 'cy_bytes': gensim.models.utils_any2vec._ft_hash_bytes_cy, } try: fun = hashmap[alg] From f392d808bfe8dbebb3486268cc9bd68d04d9f601 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Tue, 22 Jan 2019 22:07:25 +1100 Subject: [PATCH 13/32] fixup: fix test_utils.py --- gensim/test/test_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gensim/test/test_utils.py b/gensim/test/test_utils.py index 27fde0e20d..32f2f16ba1 100644 --- a/gensim/test/test_utils.py +++ b/gensim/test/test_utils.py @@ -335,21 +335,21 @@ def setUp(self): } def test_python(self): - actual = {k: gensim.models.utils_any2vec._ft_hash_py_bytes(k.encode('utf-8')) for k in self.expected} + actual = {k: gensim.models.utils_any2vec._ft_hash_bytes_py(k.encode('utf-8')) for k in self.expected} self.assertEqual(self.expected, actual) @unittest.skipIf(DISABLE_CYTHON_TESTS, 'Cython functions are not properly compiled') def test_cython(self): - actual = {k: gensim.models.utils_any2vec._ft_hash_cy_bytes(k.encode('utf-8')) for k in self.expected} + actual = {k: gensim.models.utils_any2vec._ft_hash_bytes_cy(k.encode('utf-8')) for k in self.expected} self.assertEqual(self.expected, actual) def test_python_broken(self): - actual = {k: gensim.models.utils_any2vec._ft_hash_py_broken(k) for k in self.expected} + actual = {k: gensim.models.utils_any2vec._ft_hash_broken_py(k) for k in self.expected} self.assertEqual(self.expected_broken, actual) @unittest.skipIf(DISABLE_CYTHON_TESTS, 'Cython functions are not properly compiled') def test_cython_broken(self): - actual = {k: gensim.models.utils_any2vec._ft_hash_cy_broken(k) for k in self.expected} + actual = {k: gensim.models.utils_any2vec._ft_hash_broken_cy(k) for k in self.expected} self.assertEqual(self.expected_broken, actual) From 4fd56d5f37568bdfae29c84d41d178e7ea5ce97b Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Tue, 22 Jan 2019 22:25:53 +1100 Subject: [PATCH 14/32] add tests for ngram generation --- gensim/test/test_utils.py | 66 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/gensim/test/test_utils.py b/gensim/test/test_utils.py index 32f2f16ba1..f7dfdd7027 100644 --- a/gensim/test/test_utils.py +++ b/gensim/test/test_utils.py @@ -6,7 +6,7 @@ """ Automated tests for checking various utils functions. """ - +from __future__ import unicode_literals import logging import unittest @@ -353,6 +353,70 @@ def test_cython_broken(self): self.assertEqual(self.expected_broken, actual) +# +# Run with: +# +# python -c 'import gensim.test.test_utils as t;t.ngram_main()' py_text 3 5 +# +def ngram_main(): + """Generate ngrams for tests from standard input.""" + import sys + import six + + alg = sys.argv[1] + minn = int(sys.argv[2]) + maxn = int(sys.argv[3]) + + assert six.PY3, 'this only works under Py3' + assert not DISABLE_CYTHON_TESTS, 'this only works if Cython extensions available' + assert minn <= maxn, 'expected sane command-line parameters' + + hashmap = { + 'py_text': gensim.models.utils_any2vec._compute_ngrams_py, + 'py_bytes': gensim.models.utils_any2vec._compute_ngrams_bytes_py, + 'cy_text': gensim.models.utils_any2vec._compute_ngrams_cy, + 'cy_bytes': gensim.models.utils_any2vec._compute_ngrams_bytes_cy, + } + try: + fun = hashmap[alg] + except KeyError: + raise KeyError('invalid alg: %r expected one of %r' % (alg, sorted(hashmap))) + + for line in sys.stdin: + word = line.rstrip('\n') + ngrams = fun(word, minn, maxn) + print("%r: %r," % (word, ngrams)) + + +class NgramsTest(unittest.TestCase): + def setUp(self): + self.expected_text = { + 'test': ['', '', ''], + 'at the': [ + '', + '', '' + ], + 'at\nthe': [ + '', + '', '' + ], + 'тест': ['<те', 'тес', 'ест', 'ст>', '<тес', 'тест', 'ест>', '<тест', 'тест>'], + 'テスト': ['<テス', 'テスト', 'スト>', '<テスト', 'テスト>', '<テスト>'], + '試し': ['<試し', '試し>', '<試し>'], + } + + def test_text_py(self): + for word in self.expected_text: + expected = self.expected_text[word] + actual = gensim.models.utils_any2vec._compute_ngrams_py(word, 3, 5) + + @unittest.skipIf(DISABLE_CYTHON_TESTS, 'Cython functions are not properly compiled') + def test_text_cy(self): + for word in self.expected_text: + expected = self.expected_text[word] + actual = gensim.models.utils_any2vec._compute_ngrams_cy(word, 3, 5) + + if __name__ == '__main__': logging.root.setLevel(logging.WARNING) unittest.main() From b614030f2689258dc9663960aa2059a8320debe3 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Tue, 22 Jan 2019 22:28:49 +1100 Subject: [PATCH 15/32] fixup in tests --- gensim/test/test_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gensim/test/test_utils.py b/gensim/test/test_utils.py index f7dfdd7027..2df746187f 100644 --- a/gensim/test/test_utils.py +++ b/gensim/test/test_utils.py @@ -398,7 +398,7 @@ def setUp(self): ], 'at\nthe': [ '', - '', '' + '', '' ], 'тест': ['<те', 'тес', 'ест', 'ст>', '<тес', 'тест', 'ест>', '<тест', 'тест>'], 'テスト': ['<テス', 'テスト', 'スト>', '<テスト', 'テスト>', '<テスト>'], @@ -409,12 +409,16 @@ def test_text_py(self): for word in self.expected_text: expected = self.expected_text[word] actual = gensim.models.utils_any2vec._compute_ngrams_py(word, 3, 5) + self.assertEqual(len(expected), len(actual)) + self.assertEqual(set(expected), set(actual)) @unittest.skipIf(DISABLE_CYTHON_TESTS, 'Cython functions are not properly compiled') def test_text_cy(self): for word in self.expected_text: expected = self.expected_text[word] actual = gensim.models.utils_any2vec._compute_ngrams_cy(word, 3, 5) + self.assertEqual(len(expected), len(actual)) + self.assertEqual(set(expected), set(actual)) if __name__ == '__main__': From 46e1ec1fdec37979db72a325d9bc1ab96460db90 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Tue, 22 Jan 2019 22:35:24 +1100 Subject: [PATCH 16/32] add emoji test case --- gensim/test/test_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gensim/test/test_utils.py b/gensim/test/test_utils.py index 2df746187f..74c3613954 100644 --- a/gensim/test/test_utils.py +++ b/gensim/test/test_utils.py @@ -403,6 +403,10 @@ def setUp(self): 'тест': ['<те', 'тес', 'ест', 'ст>', '<тес', 'тест', 'ест>', '<тест', 'тест>'], 'テスト': ['<テス', 'テスト', 'スト>', '<テスト', 'テスト>', '<テスト>'], '試し': ['<試し', '試し>', '<試し>'], + '🚑🚒🚓🚕': [ + '<🚑🚒', '🚑🚒🚓', '🚒🚓🚕', '🚓🚕>', + '<🚑🚒🚓', '🚑🚒🚓🚕', '🚒🚓🚕>', '<🚑🚒🚓🚕', '🚑🚒🚓🚕>' + ], } def test_text_py(self): From d1b80c04bce81ad8b253fe01232d1babe59d5c9e Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Tue, 22 Jan 2019 22:48:18 +1100 Subject: [PATCH 17/32] minor fixup in logging message --- gensim/models/fasttext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gensim/models/fasttext.py b/gensim/models/fasttext.py index 97f5968cad..807ebec32d 100644 --- a/gensim/models/fasttext.py +++ b/gensim/models/fasttext.py @@ -713,7 +713,7 @@ def train(self, sentences=None, corpus_file=None, total_examples=None, total_wor raise ValueError( 'this model cannot be trained any further, ' 'if this is a native model, try loading it with ' - 'with FastText.load_fasttext_format(path, full_model=True)' + 'FastText.load_fasttext_format(path, full_model=True)' ) super(FastText, self).train( From 07c49c2f452e75cf5e6562b0a96239cb91c4cbb4 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Tue, 22 Jan 2019 23:08:54 +1100 Subject: [PATCH 18/32] add byte tests --- gensim/test/test_utils.py | 60 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/gensim/test/test_utils.py b/gensim/test/test_utils.py index 74c3613954..82d33470da 100644 --- a/gensim/test/test_utils.py +++ b/gensim/test/test_utils.py @@ -408,21 +408,73 @@ def setUp(self): '<🚑🚒🚓', '🚑🚒🚓🚕', '🚒🚓🚕>', '<🚑🚒🚓🚕', '🚑🚒🚓🚕>' ], } + self.expected_bytes = { + 'test': [b'', b'est', b'est>', b'st>'], + 'at the': [ + b'', b'the', b'the>', b'he>' + ], + 'тест': [ + b'<\xd1\x82\xd0\xb5', b'<\xd1\x82\xd0\xb5\xd1\x81', b'<\xd1\x82\xd0\xb5\xd1\x81\xd1\x82', + b'\xd1\x82\xd0\xb5\xd1\x81', b'\xd1\x82\xd0\xb5\xd1\x81\xd1\x82', b'\xd1\x82\xd0\xb5\xd1\x81\xd1\x82>', + b'\xd0\xb5\xd1\x81\xd1\x82', b'\xd0\xb5\xd1\x81\xd1\x82>', b'\xd1\x81\xd1\x82>' + ], + 'テスト': [ + b'<\xe3\x83\x86\xe3\x82\xb9', b'<\xe3\x83\x86\xe3\x82\xb9\xe3\x83\x88', + b'<\xe3\x83\x86\xe3\x82\xb9\xe3\x83\x88>', b'\xe3\x83\x86\xe3\x82\xb9\xe3\x83\x88', + b'\xe3\x83\x86\xe3\x82\xb9\xe3\x83\x88>',b'\xe3\x82\xb9\xe3\x83\x88>' + ], + '試し': [b'<\xe8\xa9\xa6\xe3\x81\x97', b'<\xe8\xa9\xa6\xe3\x81\x97>', b'\xe8\xa9\xa6\xe3\x81\x97>'], + '🚑🚒🚓🚕': [ + b'<\xf0\x9f\x9a\x91\xf0\x9f\x9a\x92', + b'<\xf0\x9f\x9a\x91\xf0\x9f\x9a\x92\xf0\x9f\x9a\x93', + b'<\xf0\x9f\x9a\x91\xf0\x9f\x9a\x92\xf0\x9f\x9a\x93\xf0\x9f\x9a\x95', + b'\xf0\x9f\x9a\x91\xf0\x9f\x9a\x92\xf0\x9f\x9a\x93', + b'\xf0\x9f\x9a\x91\xf0\x9f\x9a\x92\xf0\x9f\x9a\x93\xf0\x9f\x9a\x95', + b'\xf0\x9f\x9a\x91\xf0\x9f\x9a\x92\xf0\x9f\x9a\x93\xf0\x9f\x9a\x95>', + b'\xf0\x9f\x9a\x92\xf0\x9f\x9a\x93\xf0\x9f\x9a\x95', + b'\xf0\x9f\x9a\x92\xf0\x9f\x9a\x93\xf0\x9f\x9a\x95>', + b'\xf0\x9f\x9a\x93\xf0\x9f\x9a\x95>' + ], + } def test_text_py(self): for word in self.expected_text: expected = self.expected_text[word] actual = gensim.models.utils_any2vec._compute_ngrams_py(word, 3, 5) - self.assertEqual(len(expected), len(actual)) - self.assertEqual(set(expected), set(actual)) + self.assertEqual(expected, actual) @unittest.skipIf(DISABLE_CYTHON_TESTS, 'Cython functions are not properly compiled') def test_text_cy(self): for word in self.expected_text: expected = self.expected_text[word] actual = gensim.models.utils_any2vec._compute_ngrams_cy(word, 3, 5) - self.assertEqual(len(expected), len(actual)) - self.assertEqual(set(expected), set(actual)) + self.assertEqual(expected, actual) + + def test_bytes_py(self): + for word in self.expected_bytes: + expected = self.expected_bytes[word] + actual = gensim.models.utils_any2vec._compute_ngrams_bytes_py(word, 3, 5) + self.assertEqual(expected, actual) + + expected_text = self.expected_text[word] + actual_text = [n.decode('utf-8') for n in actual] + # + # The text and byte implementations yield ngrams in different + # order, so the test ignores ngram order. + # + self.assertEqual(sorted(set(expected_text)), sorted(set(actual_text))) + + @unittest.skipIf(DISABLE_CYTHON_TESTS, 'Cython functions are not properly compiled') + def test_bytes_cy(self): + for word in self.expected_bytes: + expected = self.expected_bytes[word] + actual = gensim.models.utils_any2vec._compute_ngrams_bytes_cy(word, 3, 5) + self.assertEqual(expected, actual) + + expected_text = self.expected_text[word] + actual_text = [n.decode('utf-8') for n in actual] + self.assertEqual(sorted(set(expected_text)), sorted(set(actual_text))) if __name__ == '__main__': From ccaba0819297d0d409dfc89a563905c9f65d30cb Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Tue, 22 Jan 2019 23:09:20 +1100 Subject: [PATCH 19/32] remove FIXME, absense of ord does not influence correctness --- gensim/models/_utils_any2vec.pyx | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/gensim/models/_utils_any2vec.pyx b/gensim/models/_utils_any2vec.pyx index 187b1905bd..cecd3b0707 100644 --- a/gensim/models/_utils_any2vec.pyx +++ b/gensim/models/_utils_any2vec.pyx @@ -19,15 +19,6 @@ import numpy as np cimport numpy as np -cdef _byte_to_int_py3(b): - return b - -cdef _byte_to_int_py2(b): - return ord(b) - -_byte_to_int = _byte_to_int_py2 if PY2 else _byte_to_int_py3 - - cpdef ft_hash_bytes(bytes bytez): """Calculate hash based on `bytez`. Reproduce `hash method from Facebook fastText implementation @@ -48,7 +39,7 @@ cpdef ft_hash_bytes(bytes bytez): cdef char b for b in bytez: - h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss + h = h ^ (b) h = h * 16777619 return h From 0b0e46ea8f45b64d86e0befd9a140898ed136971 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Wed, 23 Jan 2019 08:28:33 +1100 Subject: [PATCH 20/32] review response: introduce list slicing --- gensim/models/_utils_any2vec.c | 1705 +++++------------------------- gensim/models/_utils_any2vec.pyx | 5 +- gensim/models/utils_any2vec.py | 7 +- 3 files changed, 280 insertions(+), 1437 deletions(-) diff --git a/gensim/models/_utils_any2vec.c b/gensim/models/_utils_any2vec.c index e66d4bd506..6e7118f529 100644 --- a/gensim/models/_utils_any2vec.c +++ b/gensim/models/_utils_any2vec.c @@ -844,7 +844,6 @@ static const char *__pyx_filename; static const char *__pyx_f[] = { "gensim/models/_utils_any2vec.pyx", "__init__.pxd", - "stringsource", "type.pxd", }; @@ -1062,7 +1061,6 @@ static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(do /*--- Type declarations ---*/ -struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py; /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":815 * ctypedef npy_longdouble longdouble_t @@ -1100,19 +1098,6 @@ typedef npy_clongdouble __pyx_t_5numpy_clongdouble_t; */ typedef npy_cdouble __pyx_t_5numpy_complex_t; -/* "cfunc.to_py":64 - * - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< - * def wrap(object b): - * """wrap(b)""" - */ -struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py { - PyObject_HEAD - PyObject *(*__pyx_v_f)(PyObject *); -}; - - /* --- Runtime support code (head) --- */ /* Refnanny.proto */ #ifndef CYTHON_REFNANNY @@ -1187,6 +1172,16 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject /* GetBuiltinName.proto */ static PyObject *__Pyx_GetBuiltinName(PyObject *name); +/* ArgTypeTest.proto */ +#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ + ((likely((Py_TYPE(obj) == type) | (none_allowed && (obj == Py_None)))) ? 1 :\ + __Pyx__ArgTypeTest(obj, type, name, exact)) +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); + +/* unicode_iter.proto */ +static CYTHON_INLINE int __Pyx_init_unicode_iteration( + PyObject* ustring, Py_ssize_t *length, void** data, int *kind); + /* UnicodeAsUCS4.proto */ static CYTHON_INLINE Py_UCS4 __Pyx_PyUnicode_AsPy_UCS4(PyObject*); @@ -1199,16 +1194,6 @@ static CYTHON_INLINE Py_UCS4 __Pyx_PyUnicode_AsPy_UCS4(PyObject*); #endif static long __Pyx__PyObject_Ord(PyObject* c); -/* ArgTypeTest.proto */ -#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ - ((likely((Py_TYPE(obj) == type) | (none_allowed && (obj == Py_None)))) ? 1 :\ - __Pyx__ArgTypeTest(obj, type, name, exact)) -static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); - -/* unicode_iter.proto */ -static CYTHON_INLINE int __Pyx_init_unicode_iteration( - PyObject* ustring, Py_ssize_t *length, void** data, int *kind); - /* PyObjectFormatSimple.proto */ #if CYTHON_COMPILING_IN_PYPY #define __Pyx_PyObject_FormatSimple(s, f) (\ @@ -1461,70 +1446,6 @@ static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); #endif -/* FetchCommonType.proto */ -static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); - -/* CythonFunction.proto */ -#define __Pyx_CyFunction_USED 1 -#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 -#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 -#define __Pyx_CYFUNCTION_CCLASS 0x04 -#define __Pyx_CyFunction_GetClosure(f)\ - (((__pyx_CyFunctionObject *) (f))->func_closure) -#define __Pyx_CyFunction_GetClassObj(f)\ - (((__pyx_CyFunctionObject *) (f))->func_classobj) -#define __Pyx_CyFunction_Defaults(type, f)\ - ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) -#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ - ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) -typedef struct { - PyCFunctionObject func; -#if PY_VERSION_HEX < 0x030500A0 - PyObject *func_weakreflist; -#endif - PyObject *func_dict; - PyObject *func_name; - PyObject *func_qualname; - PyObject *func_doc; - PyObject *func_globals; - PyObject *func_code; - PyObject *func_closure; - PyObject *func_classobj; - void *defaults; - int defaults_pyobjects; - int flags; - PyObject *defaults_tuple; - PyObject *defaults_kwdict; - PyObject *(*defaults_getter)(PyObject *); - PyObject *func_annotations; -} __pyx_CyFunctionObject; -static PyTypeObject *__pyx_CyFunctionType = 0; -#define __Pyx_CyFunction_Check(obj) (__Pyx_TypeCheck(obj, __pyx_CyFunctionType)) -#define __Pyx_CyFunction_NewEx(ml, flags, qualname, self, module, globals, code)\ - __Pyx_CyFunction_New(__pyx_CyFunctionType, ml, flags, qualname, self, module, globals, code) -static PyObject *__Pyx_CyFunction_New(PyTypeObject *, PyMethodDef *ml, - int flags, PyObject* qualname, - PyObject *self, - PyObject *module, PyObject *globals, - PyObject* code); -static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *m, - size_t size, - int pyobjects); -static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, - PyObject *tuple); -static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, - PyObject *dict); -static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, - PyObject *dict); -static int __pyx_CyFunction_init(void); - -/* PyObject_GenericGetAttrNoDict.proto */ -#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 -static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name); -#else -#define __Pyx_PyObject_GenericGetAttrNoDict PyObject_GenericGetAttr -#endif - /* TypeImport.proto */ #ifndef __PYX_HAVE_RT_ImportType_proto #define __PYX_HAVE_RT_ImportType_proto @@ -1568,9 +1489,6 @@ static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); static void __Pyx_AddTraceback(const char *funcname, int c_line, int py_line, const char *filename); -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); - /* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint32_t(uint32_t value); @@ -1690,6 +1608,9 @@ static CYTHON_INLINE char __Pyx_PyInt_As_char(PyObject *); /* CIntFromPy.proto */ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); + /* CIntFromPy.proto */ static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); @@ -1743,14 +1664,10 @@ static PyTypeObject *__pyx_ptype_5numpy_ufunc = 0; static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *, char *, char *, int *); /*proto*/ /* Module declarations from 'gensim.models._utils_any2vec' */ -static PyTypeObject *__pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py = 0; -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3(PyObject *); /*proto*/ -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObject *); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject *, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObject *, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObject *, unsigned int, unsigned int, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(PyObject *, unsigned int, unsigned int, int __pyx_skip_dispatch); /*proto*/ -static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*)(PyObject *)); /*proto*/ #define __Pyx_MODULE_NAME "gensim.models._utils_any2vec" extern int __pyx_module_is_main_gensim__models___utils_any2vec; int __pyx_module_is_main_gensim__models___utils_any2vec = 0; @@ -1761,7 +1678,6 @@ static PyObject *__pyx_builtin_ValueError; static PyObject *__pyx_builtin_RuntimeError; static PyObject *__pyx_builtin_ImportError; static const char __pyx_k_[] = "<"; -static const char __pyx_k_b[] = "b"; static const char __pyx_k__2[] = ">"; static const char __pyx_k_np[] = "np"; static const char __pyx_k_PY2[] = "PY2"; @@ -1770,7 +1686,6 @@ static const char __pyx_k_main[] = "__main__"; static const char __pyx_k_name[] = "__name__"; static const char __pyx_k_test[] = "__test__"; static const char __pyx_k_word[] = "word"; -static const char __pyx_k_wrap[] = "wrap"; static const char __pyx_k_max_n[] = "max_n"; static const char __pyx_k_min_n[] = "min_n"; static const char __pyx_k_numpy[] = "numpy"; @@ -1780,13 +1695,9 @@ static const char __pyx_k_MB_MASK[] = "_MB_MASK"; static const char __pyx_k_MB_START[] = "_MB_START"; static const char __pyx_k_ValueError[] = "ValueError"; static const char __pyx_k_ImportError[] = "ImportError"; -static const char __pyx_k_byte_to_int[] = "_byte_to_int"; -static const char __pyx_k_cfunc_to_py[] = "cfunc.to_py"; static const char __pyx_k_RuntimeError[] = "RuntimeError"; -static const char __pyx_k_stringsource[] = "stringsource"; static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; static const char __pyx_k_ndarray_is_not_C_contiguous[] = "ndarray is not C contiguous"; -static const char __pyx_k_Pyx_CFunc_object____object___t[] = "__Pyx_CFunc_object____object___to_py..wrap"; static const char __pyx_k_numpy_core_multiarray_failed_to[] = "numpy.core.multiarray failed to import"; static const char __pyx_k_unknown_dtype_code_in_numpy_pxd[] = "unknown dtype code in numpy.pxd (%d)"; static const char __pyx_k_Format_string_allocated_too_shor[] = "Format string allocated too short, see comment in numpy.pxd"; @@ -1803,13 +1714,9 @@ static PyObject *__pyx_n_s_MB_MASK; static PyObject *__pyx_n_s_MB_START; static PyObject *__pyx_kp_u_Non_native_byte_order_not_suppor; static PyObject *__pyx_n_s_PY2; -static PyObject *__pyx_n_s_Pyx_CFunc_object____object___t; static PyObject *__pyx_n_s_RuntimeError; static PyObject *__pyx_n_s_ValueError; static PyObject *__pyx_kp_u__2; -static PyObject *__pyx_n_s_b; -static PyObject *__pyx_n_s_byte_to_int; -static PyObject *__pyx_n_s_cfunc_to_py; static PyObject *__pyx_n_s_cline_in_traceback; static PyObject *__pyx_n_s_import; static PyObject *__pyx_n_s_main; @@ -1824,19 +1731,15 @@ static PyObject *__pyx_kp_s_numpy_core_multiarray_failed_to; static PyObject *__pyx_kp_s_numpy_core_umath_failed_to_impor; static PyObject *__pyx_n_s_range; static PyObject *__pyx_n_s_six; -static PyObject *__pyx_kp_s_stringsource; static PyObject *__pyx_n_s_test; static PyObject *__pyx_kp_u_unknown_dtype_code_in_numpy_pxd; static PyObject *__pyx_n_s_word; -static PyObject *__pyx_n_s_wrap; static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_bytez); /* proto */ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string); /* proto */ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info); /* proto */ -static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(PyObject *__pyx_self, PyObject *__pyx_v_b); /* proto */ -static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_int_0; static PyObject *__pyx_int_1; static PyObject *__pyx_int_128; @@ -1848,100 +1751,9 @@ static PyObject *__pyx_tuple__6; static PyObject *__pyx_tuple__7; static PyObject *__pyx_tuple__8; static PyObject *__pyx_tuple__9; -static PyObject *__pyx_tuple__10; -static PyObject *__pyx_codeobj__11; /* Late includes */ /* "gensim/models/_utils_any2vec.pyx":22 - * - * - * cdef _byte_to_int_py3(b): # <<<<<<<<<<<<<< - * return b - * - */ - -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3(PyObject *__pyx_v_b) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("_byte_to_int_py3", 0); - - /* "gensim/models/_utils_any2vec.pyx":23 - * - * cdef _byte_to_int_py3(b): - * return b # <<<<<<<<<<<<<< - * - * cdef _byte_to_int_py2(b): - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_b); - __pyx_r = __pyx_v_b; - goto __pyx_L0; - - /* "gensim/models/_utils_any2vec.pyx":22 - * - * - * cdef _byte_to_int_py3(b): # <<<<<<<<<<<<<< - * return b - * - */ - - /* function exit code */ - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "gensim/models/_utils_any2vec.pyx":25 - * return b - * - * cdef _byte_to_int_py2(b): # <<<<<<<<<<<<<< - * return ord(b) - * - */ - -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2(PyObject *__pyx_v_b) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - long __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - __Pyx_RefNannySetupContext("_byte_to_int_py2", 0); - - /* "gensim/models/_utils_any2vec.pyx":26 - * - * cdef _byte_to_int_py2(b): - * return ord(b) # <<<<<<<<<<<<<< - * - * _byte_to_int = _byte_to_int_py2 if PY2 else _byte_to_int_py3 - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_Ord(__pyx_v_b); if (unlikely(__pyx_t_1 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 26, __pyx_L1_error) - __pyx_t_2 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 26, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; - goto __pyx_L0; - - /* "gensim/models/_utils_any2vec.pyx":25 - * return b - * - * cdef _byte_to_int_py2(b): # <<<<<<<<<<<<<< - * return ord(b) - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("gensim.models._utils_any2vec._byte_to_int_py2", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "gensim/models/_utils_any2vec.pyx":31 * * * cpdef ft_hash_bytes(bytes bytez): # <<<<<<<<<<<<<< @@ -1963,7 +1775,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject PyObject *__pyx_t_6 = NULL; __Pyx_RefNannySetupContext("ft_hash_bytes", 0); - /* "gensim/models/_utils_any2vec.pyx":47 + /* "gensim/models/_utils_any2vec.pyx":38 * * """ * cdef uint32_t h = 2166136261 # <<<<<<<<<<<<<< @@ -1972,16 +1784,16 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject */ __pyx_v_h = 0x811C9DC5; - /* "gensim/models/_utils_any2vec.pyx":50 + /* "gensim/models/_utils_any2vec.pyx":41 * cdef char b * * for b in bytez: # <<<<<<<<<<<<<< - * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss + * h = h ^ (b) * h = h * 16777619 */ if (unlikely(__pyx_v_bytez == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); - __PYX_ERR(0, 50, __pyx_L1_error) + __PYX_ERR(0, 41, __pyx_L1_error) } __Pyx_INCREF(__pyx_v_bytez); __pyx_t_1 = __pyx_v_bytez; @@ -1991,18 +1803,18 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject __pyx_t_2 = __pyx_t_5; __pyx_v_b = (__pyx_t_2[0]); - /* "gensim/models/_utils_any2vec.pyx":51 + /* "gensim/models/_utils_any2vec.pyx":42 * * for b in bytez: - * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss # <<<<<<<<<<<<<< + * h = h ^ (b) # <<<<<<<<<<<<<< * h = h * 16777619 * return h */ __pyx_v_h = (__pyx_v_h ^ ((uint32_t)((int8_t)__pyx_v_b))); - /* "gensim/models/_utils_any2vec.pyx":52 + /* "gensim/models/_utils_any2vec.pyx":43 * for b in bytez: - * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss + * h = h ^ (b) * h = h * 16777619 # <<<<<<<<<<<<<< * return h * @@ -2011,21 +1823,21 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":53 - * h = h ^ (b) # FIXME I drop 'ord' from py2, not sure about correctenss + /* "gensim/models/_utils_any2vec.pyx":44 + * h = h ^ (b) * h = h * 16777619 * return h # <<<<<<<<<<<<<< * * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_6 = __Pyx_PyInt_From_uint32_t(__pyx_v_h); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 53, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyInt_From_uint32_t(__pyx_v_h); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 44, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_r = __pyx_t_6; __pyx_t_6 = 0; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":31 + /* "gensim/models/_utils_any2vec.pyx":22 * * * cpdef ft_hash_bytes(bytes bytez): # <<<<<<<<<<<<<< @@ -2052,7 +1864,7 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash_bytes(PyObje PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("ft_hash_bytes (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_bytez), (&PyBytes_Type), 1, "bytez", 1))) __PYX_ERR(0, 31, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_bytez), (&PyBytes_Type), 1, "bytez", 1))) __PYX_ERR(0, 22, __pyx_L1_error) __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(__pyx_self, ((PyObject*)__pyx_v_bytez)); /* function exit code */ @@ -2070,7 +1882,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(CYTHON_ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("ft_hash_bytes", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(__pyx_v_bytez, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 31, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(__pyx_v_bytez, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -2087,7 +1899,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(CYTHON_ return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":56 +/* "gensim/models/_utils_any2vec.pyx":47 * * * cpdef ft_hash_broken(unicode string): # <<<<<<<<<<<<<< @@ -2112,7 +1924,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec long __pyx_t_9; __Pyx_RefNannySetupContext("ft_hash_broken", 0); - /* "gensim/models/_utils_any2vec.pyx":73 + /* "gensim/models/_utils_any2vec.pyx":64 * * """ * cdef unsigned int h = 2166136261 # <<<<<<<<<<<<<< @@ -2121,7 +1933,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec */ __pyx_v_h = 0x811C9DC5; - /* "gensim/models/_utils_any2vec.pyx":74 + /* "gensim/models/_utils_any2vec.pyx":65 * """ * cdef unsigned int h = 2166136261 * for c in string: # <<<<<<<<<<<<<< @@ -2130,29 +1942,29 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec */ if (unlikely(__pyx_v_string == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); - __PYX_ERR(0, 74, __pyx_L1_error) + __PYX_ERR(0, 65, __pyx_L1_error) } __Pyx_INCREF(__pyx_v_string); __pyx_t_1 = __pyx_v_string; - __pyx_t_6 = __Pyx_init_unicode_iteration(__pyx_t_1, (&__pyx_t_3), (&__pyx_t_4), (&__pyx_t_5)); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 74, __pyx_L1_error) + __pyx_t_6 = __Pyx_init_unicode_iteration(__pyx_t_1, (&__pyx_t_3), (&__pyx_t_4), (&__pyx_t_5)); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 65, __pyx_L1_error) for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_3; __pyx_t_7++) { __pyx_t_2 = __pyx_t_7; - __pyx_t_8 = PyUnicode_FromOrdinal(__Pyx_PyUnicode_READ(__pyx_t_5, __pyx_t_4, __pyx_t_2)); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 74, __pyx_L1_error) + __pyx_t_8 = PyUnicode_FromOrdinal(__Pyx_PyUnicode_READ(__pyx_t_5, __pyx_t_4, __pyx_t_2)); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 65, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_XDECREF_SET(__pyx_v_c, __pyx_t_8); __pyx_t_8 = 0; - /* "gensim/models/_utils_any2vec.pyx":75 + /* "gensim/models/_utils_any2vec.pyx":66 * cdef unsigned int h = 2166136261 * for c in string: * h ^= ord(c) # <<<<<<<<<<<<<< * h *= 16777619 * return h */ - __pyx_t_9 = __Pyx_PyObject_Ord(__pyx_v_c); if (unlikely(__pyx_t_9 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 75, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyObject_Ord(__pyx_v_c); if (unlikely(__pyx_t_9 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 66, __pyx_L1_error) __pyx_v_h = (__pyx_v_h ^ __pyx_t_9); - /* "gensim/models/_utils_any2vec.pyx":76 + /* "gensim/models/_utils_any2vec.pyx":67 * for c in string: * h ^= ord(c) * h *= 16777619 # <<<<<<<<<<<<<< @@ -2163,7 +1975,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":77 + /* "gensim/models/_utils_any2vec.pyx":68 * h ^= ord(c) * h *= 16777619 * return h # <<<<<<<<<<<<<< @@ -2171,13 +1983,13 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObjec * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_8 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 77, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyInt_From_unsigned_int(__pyx_v_h); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 68, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __pyx_r = __pyx_t_8; __pyx_t_8 = 0; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":56 + /* "gensim/models/_utils_any2vec.pyx":47 * * * cpdef ft_hash_broken(unicode string): # <<<<<<<<<<<<<< @@ -2205,7 +2017,7 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken(PyObj PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("ft_hash_broken (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_string), (&PyUnicode_Type), 1, "string", 1))) __PYX_ERR(0, 56, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_string), (&PyUnicode_Type), 1, "string", 1))) __PYX_ERR(0, 47, __pyx_L1_error) __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(__pyx_self, ((PyObject*)__pyx_v_string)); /* function exit code */ @@ -2223,7 +2035,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHO PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("ft_hash_broken", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(__pyx_v_string, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 56, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(__pyx_v_string, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 47, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -2240,7 +2052,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHO return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":80 +/* "gensim/models/_utils_any2vec.pyx":71 * * * cpdef compute_ngrams(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -2272,14 +2084,14 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec int __pyx_t_14; __Pyx_RefNannySetupContext("compute_ngrams", 0); - /* "gensim/models/_utils_any2vec.pyx":98 + /* "gensim/models/_utils_any2vec.pyx":89 * * """ * cdef unicode extended_word = f'<{word}>' # <<<<<<<<<<<<<< * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): */ - __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 89, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = 0; __pyx_t_3 = 127; @@ -2287,7 +2099,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __pyx_t_2 += 1; __Pyx_GIVEREF(__pyx_kp_u_); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_kp_u_); - __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_word, __pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_word, __pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 89, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = (__Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) > __pyx_t_3) ? __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) : __pyx_t_3; __pyx_t_2 += __Pyx_PyUnicode_GET_LENGTH(__pyx_t_4); @@ -2298,43 +2110,43 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __pyx_t_2 += 1; __Pyx_GIVEREF(__pyx_kp_u__2); PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_kp_u__2); - __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_1, 3, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_1, 3, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 89, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_extended_word = ((PyObject*)__pyx_t_4); __pyx_t_4 = 0; - /* "gensim/models/_utils_any2vec.pyx":99 + /* "gensim/models/_utils_any2vec.pyx":90 * """ * cdef unicode extended_word = f'<{word}>' * ngrams = [] # <<<<<<<<<<<<<< * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): */ - __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 99, __pyx_L1_error) + __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 90, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_v_ngrams = ((PyObject*)__pyx_t_4); __pyx_t_4 = 0; - /* "gensim/models/_utils_any2vec.pyx":100 + /* "gensim/models/_utils_any2vec.pyx":91 * cdef unicode extended_word = f'<{word}>' * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): # <<<<<<<<<<<<<< * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) */ - __pyx_t_4 = __Pyx_PyInt_From_unsigned_int(__pyx_v_min_n); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 100, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyInt_From_unsigned_int(__pyx_v_min_n); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 91, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = __pyx_v_max_n; - __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 100, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 91, __pyx_L1_error) if (((__pyx_t_5 < __pyx_t_2) != 0)) { __pyx_t_6 = __pyx_t_5; } else { __pyx_t_6 = __pyx_t_2; } - __pyx_t_1 = PyInt_FromSsize_t((__pyx_t_6 + 1)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 100, __pyx_L1_error) + __pyx_t_1 = PyInt_FromSsize_t((__pyx_t_6 + 1)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 91, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 100, __pyx_L1_error) + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 91, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_4); @@ -2342,16 +2154,16 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_1); __pyx_t_4 = 0; __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 100, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 91, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (likely(PyList_CheckExact(__pyx_t_1)) || PyTuple_CheckExact(__pyx_t_1)) { __pyx_t_7 = __pyx_t_1; __Pyx_INCREF(__pyx_t_7); __pyx_t_6 = 0; __pyx_t_8 = NULL; } else { - __pyx_t_6 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 100, __pyx_L1_error) + __pyx_t_6 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 91, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_8 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 100, __pyx_L1_error) + __pyx_t_8 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 91, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; for (;;) { @@ -2359,17 +2171,17 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec if (likely(PyList_CheckExact(__pyx_t_7))) { if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_7)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 100, __pyx_L1_error) + __pyx_t_1 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 91, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 100, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 91, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } else { if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_7)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 100, __pyx_L1_error) + __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 91, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 100, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_7, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 91, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } @@ -2379,7 +2191,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(0, 100, __pyx_L1_error) + else __PYX_ERR(0, 91, __pyx_L1_error) } break; } @@ -2388,23 +2200,23 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __Pyx_XDECREF_SET(__pyx_v_ngram_length, __pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":101 + /* "gensim/models/_utils_any2vec.pyx":92 * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): # <<<<<<<<<<<<<< * ngrams.append(extended_word[i:i + ngram_length]) * return ngrams */ - __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 101, __pyx_L1_error) - __pyx_t_1 = PyInt_FromSsize_t(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 101, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_extended_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_t_1 = PyInt_FromSsize_t(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_4 = PyNumber_Subtract(__pyx_t_1, __pyx_v_ngram_length); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 101, __pyx_L1_error) + __pyx_t_4 = PyNumber_Subtract(__pyx_t_1, __pyx_v_ngram_length); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyInt_AddObjC(__pyx_t_4, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 101, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_AddObjC(__pyx_t_4, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 101, __pyx_L1_error) + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_INCREF(__pyx_int_0); __Pyx_GIVEREF(__pyx_int_0); @@ -2412,16 +2224,16 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 101, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (likely(PyList_CheckExact(__pyx_t_1)) || PyTuple_CheckExact(__pyx_t_1)) { __pyx_t_4 = __pyx_t_1; __Pyx_INCREF(__pyx_t_4); __pyx_t_2 = 0; __pyx_t_9 = NULL; } else { - __pyx_t_2 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 101, __pyx_L1_error) + __pyx_t_2 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_9 = Py_TYPE(__pyx_t_4)->tp_iternext; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 101, __pyx_L1_error) + __pyx_t_9 = Py_TYPE(__pyx_t_4)->tp_iternext; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 92, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; for (;;) { @@ -2429,17 +2241,17 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec if (likely(PyList_CheckExact(__pyx_t_4))) { if (__pyx_t_2 >= PyList_GET_SIZE(__pyx_t_4)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 101, __pyx_L1_error) + __pyx_t_1 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 92, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 101, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } else { if (__pyx_t_2 >= PyTuple_GET_SIZE(__pyx_t_4)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 101, __pyx_L1_error) + __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(0, 92, __pyx_L1_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 101, __pyx_L1_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif } @@ -2449,7 +2261,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(0, 101, __pyx_L1_error) + else __PYX_ERR(0, 92, __pyx_L1_error) } break; } @@ -2458,7 +2270,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __Pyx_XDECREF_SET(__pyx_v_i, __pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":102 + /* "gensim/models/_utils_any2vec.pyx":93 * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) # <<<<<<<<<<<<<< @@ -2471,26 +2283,26 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec if (__pyx_t_11) { __pyx_t_10 = 0; } else { - __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 102, __pyx_L1_error) + __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 93, __pyx_L1_error) __pyx_t_10 = __pyx_t_12; } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyNumber_Add(__pyx_v_i, __pyx_v_ngram_length); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 102, __pyx_L1_error) + __pyx_t_1 = PyNumber_Add(__pyx_v_i, __pyx_v_ngram_length); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 93, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_11 = (__pyx_t_1 == Py_None); if (__pyx_t_11) { __pyx_t_12 = PY_SSIZE_T_MAX; } else { - __pyx_t_13 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_13 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 102, __pyx_L1_error) + __pyx_t_13 = __Pyx_PyIndex_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_13 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 93, __pyx_L1_error) __pyx_t_12 = __pyx_t_13; } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyUnicode_Substring(__pyx_v_extended_word, __pyx_t_10, __pyx_t_12); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 102, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyUnicode_Substring(__pyx_v_extended_word, __pyx_t_10, __pyx_t_12); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 93, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_1); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 102, __pyx_L1_error) + __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_1); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 93, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":101 + /* "gensim/models/_utils_any2vec.pyx":92 * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): * for i in range(0, len(extended_word) - ngram_length + 1): # <<<<<<<<<<<<<< @@ -2500,7 +2312,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec } __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "gensim/models/_utils_any2vec.pyx":100 + /* "gensim/models/_utils_any2vec.pyx":91 * cdef unicode extended_word = f'<{word}>' * ngrams = [] * for ngram_length in range(min_n, min(len(extended_word), max_n) + 1): # <<<<<<<<<<<<<< @@ -2510,7 +2322,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - /* "gensim/models/_utils_any2vec.pyx":103 + /* "gensim/models/_utils_any2vec.pyx":94 * for i in range(0, len(extended_word) - ngram_length + 1): * ngrams.append(extended_word[i:i + ngram_length]) * return ngrams # <<<<<<<<<<<<<< @@ -2522,7 +2334,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec __pyx_r = __pyx_v_ngrams; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":80 + /* "gensim/models/_utils_any2vec.pyx":71 * * * cpdef compute_ngrams(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -2582,17 +2394,17 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams(PyObj case 1: if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_min_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 1); __PYX_ERR(0, 80, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 1); __PYX_ERR(0, 71, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 2); __PYX_ERR(0, 80, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, 2); __PYX_ERR(0, 71, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams") < 0)) __PYX_ERR(0, 80, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams") < 0)) __PYX_ERR(0, 71, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { goto __pyx_L5_argtuple_error; @@ -2602,12 +2414,12 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams(PyObj values[2] = PyTuple_GET_ITEM(__pyx_args, 2); } __pyx_v_word = values[0]; - __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 80, __pyx_L3_error) - __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 80, __pyx_L3_error) + __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 71, __pyx_L3_error) + __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 71, __pyx_L3_error) } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 80, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 71, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -2626,7 +2438,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHO PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("compute_ngrams", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 80, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 71, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -2643,7 +2455,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHO return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":113 +/* "gensim/models/_utils_any2vec.pyx":104 * * * cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -2678,14 +2490,14 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P int __pyx_t_14; __Pyx_RefNannySetupContext("compute_ngrams_bytes", 0); - /* "gensim/models/_utils_any2vec.pyx":137 + /* "gensim/models/_utils_any2vec.pyx":128 * * """ * cdef bytes utf8_word = f'<{word}>'.encode("utf-8") # <<<<<<<<<<<<<< * cdef int num_bytes = len(utf8_word) * cdef int j, i, n */ - __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 137, __pyx_L1_error) + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 128, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = 0; __pyx_t_3 = 127; @@ -2693,7 +2505,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_t_2 += 1; __Pyx_GIVEREF(__pyx_kp_u_); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_kp_u_); - __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_word, __pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 137, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_word, __pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 128, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = (__Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) > __pyx_t_3) ? __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) : __pyx_t_3; __pyx_t_2 += __Pyx_PyUnicode_GET_LENGTH(__pyx_t_4); @@ -2704,16 +2516,16 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_t_2 += 1; __Pyx_GIVEREF(__pyx_kp_u__2); PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_kp_u__2); - __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_1, 3, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 137, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_1, 3, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 128, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyUnicode_AsUTF8String(((PyObject*)__pyx_t_4)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 137, __pyx_L1_error) + __pyx_t_1 = PyUnicode_AsUTF8String(((PyObject*)__pyx_t_4)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 128, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_v_utf8_word = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":138 + /* "gensim/models/_utils_any2vec.pyx":129 * """ * cdef bytes utf8_word = f'<{word}>'.encode("utf-8") * cdef int num_bytes = len(utf8_word) # <<<<<<<<<<<<<< @@ -2722,73 +2534,61 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P */ if (unlikely(__pyx_v_utf8_word == Py_None)) { PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); - __PYX_ERR(0, 138, __pyx_L1_error) + __PYX_ERR(0, 129, __pyx_L1_error) } - __pyx_t_2 = PyBytes_GET_SIZE(__pyx_v_utf8_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 138, __pyx_L1_error) + __pyx_t_2 = PyBytes_GET_SIZE(__pyx_v_utf8_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 129, __pyx_L1_error) __pyx_v_num_bytes = __pyx_t_2; - /* "gensim/models/_utils_any2vec.pyx":141 + /* "gensim/models/_utils_any2vec.pyx":132 * cdef int j, i, n * * ngrams = [] # <<<<<<<<<<<<<< * for i in range(num_bytes): - * ngram = [] + * if utf8_word[i] & _MB_MASK == _MB_START: */ - __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 141, __pyx_L1_error) + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 132, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_ngrams = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":142 + /* "gensim/models/_utils_any2vec.pyx":133 * * ngrams = [] * for i in range(num_bytes): # <<<<<<<<<<<<<< - * ngram = [] - * + * if utf8_word[i] & _MB_MASK == _MB_START: + * continue */ __pyx_t_5 = __pyx_v_num_bytes; __pyx_t_6 = __pyx_t_5; for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_6; __pyx_t_7+=1) { __pyx_v_i = __pyx_t_7; - /* "gensim/models/_utils_any2vec.pyx":143 + /* "gensim/models/_utils_any2vec.pyx":134 * ngrams = [] * for i in range(num_bytes): - * ngram = [] # <<<<<<<<<<<<<< - * - * if utf8_word[i] & _MB_MASK == _MB_START: - */ - __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 143, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_XDECREF_SET(__pyx_v_ngram, ((PyObject*)__pyx_t_1)); - __pyx_t_1 = 0; - - /* "gensim/models/_utils_any2vec.pyx":145 - * ngram = [] - * * if utf8_word[i] & _MB_MASK == _MB_START: # <<<<<<<<<<<<<< * continue * */ - __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 145, __pyx_L1_error) + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 134, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 145, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 134, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_8 = PyNumber_And(__pyx_t_1, __pyx_t_4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 145, __pyx_L1_error) + __pyx_t_8 = PyNumber_And(__pyx_t_1, __pyx_t_4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 134, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 145, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 134, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_1 = PyObject_RichCompare(__pyx_t_8, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 145, __pyx_L1_error) + __pyx_t_1 = PyObject_RichCompare(__pyx_t_8, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 134, __pyx_L1_error) __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_9 < 0)) __PYX_ERR(0, 145, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_9 < 0)) __PYX_ERR(0, 134, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_9) { - /* "gensim/models/_utils_any2vec.pyx":146 - * + /* "gensim/models/_utils_any2vec.pyx":135 + * for i in range(num_bytes): * if utf8_word[i] & _MB_MASK == _MB_START: * continue # <<<<<<<<<<<<<< * @@ -2796,33 +2596,33 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P */ goto __pyx_L3_continue; - /* "gensim/models/_utils_any2vec.pyx":145 - * ngram = [] - * + /* "gensim/models/_utils_any2vec.pyx":134 + * ngrams = [] + * for i in range(num_bytes): * if utf8_word[i] & _MB_MASK == _MB_START: # <<<<<<<<<<<<<< * continue * */ } - /* "gensim/models/_utils_any2vec.pyx":148 + /* "gensim/models/_utils_any2vec.pyx":137 * continue * * j, n = i, 1 # <<<<<<<<<<<<<< * while j < num_bytes and n <= max_n: - * ngram.append(utf8_word[j]) + * j += 1 */ __pyx_t_10 = __pyx_v_i; __pyx_t_11 = 1; __pyx_v_j = __pyx_t_10; __pyx_v_n = __pyx_t_11; - /* "gensim/models/_utils_any2vec.pyx":149 + /* "gensim/models/_utils_any2vec.pyx":138 * * j, n = i, 1 * while j < num_bytes and n <= max_n: # <<<<<<<<<<<<<< - * ngram.append(utf8_word[j]) * j += 1 + * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: */ while (1) { __pyx_t_12 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); @@ -2836,33 +2636,21 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_L8_bool_binop_done:; if (!__pyx_t_9) break; - /* "gensim/models/_utils_any2vec.pyx":150 + /* "gensim/models/_utils_any2vec.pyx":139 * j, n = i, 1 * while j < num_bytes and n <= max_n: - * ngram.append(utf8_word[j]) # <<<<<<<<<<<<<< - * j += 1 - * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: - */ - __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 150, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_13 = __Pyx_PyList_Append(__pyx_v_ngram, __pyx_t_1); if (unlikely(__pyx_t_13 == ((int)-1))) __PYX_ERR(0, 150, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "gensim/models/_utils_any2vec.pyx":151 - * while j < num_bytes and n <= max_n: - * ngram.append(utf8_word[j]) * j += 1 # <<<<<<<<<<<<<< * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: - * ngram.append(utf8_word[j]) + * j += 1 */ __pyx_v_j = (__pyx_v_j + 1); - /* "gensim/models/_utils_any2vec.pyx":152 - * ngram.append(utf8_word[j]) + /* "gensim/models/_utils_any2vec.pyx":140 + * while j < num_bytes and n <= max_n: * j += 1 * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: # <<<<<<<<<<<<<< - * ngram.append(utf8_word[j]) * j += 1 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): */ while (1) { __pyx_t_12 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); @@ -2871,53 +2659,41 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_t_9 = __pyx_t_12; goto __pyx_L12_bool_binop_done; } - __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 152, __pyx_L1_error) + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 140, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 152, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 140, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_8 = PyNumber_And(__pyx_t_1, __pyx_t_4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 152, __pyx_L1_error) + __pyx_t_8 = PyNumber_And(__pyx_t_1, __pyx_t_4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 140, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 152, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 140, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_1 = PyObject_RichCompare(__pyx_t_8, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 152, __pyx_L1_error) + __pyx_t_1 = PyObject_RichCompare(__pyx_t_8, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 140, __pyx_L1_error) __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_12 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_12 < 0)) __PYX_ERR(0, 152, __pyx_L1_error) + __pyx_t_12 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_12 < 0)) __PYX_ERR(0, 140, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_9 = __pyx_t_12; __pyx_L12_bool_binop_done:; if (!__pyx_t_9) break; - /* "gensim/models/_utils_any2vec.pyx":153 + /* "gensim/models/_utils_any2vec.pyx":141 * j += 1 * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: - * ngram.append(utf8_word[j]) # <<<<<<<<<<<<<< - * j += 1 - * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): - */ - __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 153, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_13 = __Pyx_PyList_Append(__pyx_v_ngram, __pyx_t_1); if (unlikely(__pyx_t_13 == ((int)-1))) __PYX_ERR(0, 153, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "gensim/models/_utils_any2vec.pyx":154 - * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: - * ngram.append(utf8_word[j]) * j += 1 # <<<<<<<<<<<<<< * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): - * ngrams.append(bytes(ngram)) + * ngram = bytes(utf8_word[i:j]) */ __pyx_v_j = (__pyx_v_j + 1); } - /* "gensim/models/_utils_any2vec.pyx":155 - * ngram.append(utf8_word[j]) + /* "gensim/models/_utils_any2vec.pyx":142 + * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< + * ngram = bytes(utf8_word[i:j]) * ngrams.append(bytes(ngram)) - * n += 1 */ __pyx_t_12 = ((__pyx_v_n >= __pyx_v_min_n) != 0); if (__pyx_t_12) { @@ -2925,49 +2701,68 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_t_9 = __pyx_t_12; goto __pyx_L15_bool_binop_done; } - __pyx_t_14 = ((__pyx_v_n == 1) != 0); - if (__pyx_t_14) { + __pyx_t_13 = ((__pyx_v_n == 1) != 0); + if (__pyx_t_13) { } else { - __pyx_t_12 = __pyx_t_14; + __pyx_t_12 = __pyx_t_13; goto __pyx_L17_bool_binop_done; } - __pyx_t_14 = ((__pyx_v_i == 0) != 0); - if (!__pyx_t_14) { + __pyx_t_13 = ((__pyx_v_i == 0) != 0); + if (!__pyx_t_13) { } else { - __pyx_t_12 = __pyx_t_14; + __pyx_t_12 = __pyx_t_13; goto __pyx_L17_bool_binop_done; } - __pyx_t_14 = ((__pyx_v_j == __pyx_v_num_bytes) != 0); - __pyx_t_12 = __pyx_t_14; + __pyx_t_13 = ((__pyx_v_j == __pyx_v_num_bytes) != 0); + __pyx_t_12 = __pyx_t_13; __pyx_L17_bool_binop_done:; - __pyx_t_14 = ((!__pyx_t_12) != 0); - __pyx_t_9 = __pyx_t_14; + __pyx_t_13 = ((!__pyx_t_12) != 0); + __pyx_t_9 = __pyx_t_13; __pyx_L15_bool_binop_done:; if (__pyx_t_9) { - /* "gensim/models/_utils_any2vec.pyx":156 + /* "gensim/models/_utils_any2vec.pyx":143 * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): - * ngrams.append(bytes(ngram)) # <<<<<<<<<<<<<< + * ngram = bytes(utf8_word[i:j]) # <<<<<<<<<<<<<< + * ngrams.append(bytes(ngram)) * n += 1 - * return ngrams */ - __pyx_t_1 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_v_ngram); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 156, __pyx_L1_error) + if (unlikely(__pyx_v_utf8_word == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 143, __pyx_L1_error) + } + __pyx_t_1 = PySequence_GetSlice(__pyx_v_utf8_word, __pyx_v_i, __pyx_v_j); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 143, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_13 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_1); if (unlikely(__pyx_t_13 == ((int)-1))) __PYX_ERR(0, 156, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 143, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF_SET(__pyx_v_ngram, ((PyObject*)__pyx_t_4)); + __pyx_t_4 = 0; + + /* "gensim/models/_utils_any2vec.pyx":144 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + * ngram = bytes(utf8_word[i:j]) + * ngrams.append(bytes(ngram)) # <<<<<<<<<<<<<< + * n += 1 + * return ngrams + */ + __pyx_t_4 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_v_ngram); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 144, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_4); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 144, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "gensim/models/_utils_any2vec.pyx":155 - * ngram.append(utf8_word[j]) + /* "gensim/models/_utils_any2vec.pyx":142 + * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< + * ngram = bytes(utf8_word[i:j]) * ngrams.append(bytes(ngram)) - * n += 1 */ } - /* "gensim/models/_utils_any2vec.pyx":157 - * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + /* "gensim/models/_utils_any2vec.pyx":145 + * ngram = bytes(utf8_word[i:j]) * ngrams.append(bytes(ngram)) * n += 1 # <<<<<<<<<<<<<< * return ngrams @@ -2977,7 +2772,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_L3_continue:; } - /* "gensim/models/_utils_any2vec.pyx":158 + /* "gensim/models/_utils_any2vec.pyx":146 * ngrams.append(bytes(ngram)) * n += 1 * return ngrams # <<<<<<<<<<<<<< @@ -2987,7 +2782,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_r = __pyx_v_ngrams; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":113 + /* "gensim/models/_utils_any2vec.pyx":104 * * * cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -3046,17 +2841,17 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes case 1: if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_min_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 1); __PYX_ERR(0, 113, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 1); __PYX_ERR(0, 104, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 2); __PYX_ERR(0, 113, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 2); __PYX_ERR(0, 104, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams_bytes") < 0)) __PYX_ERR(0, 113, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams_bytes") < 0)) __PYX_ERR(0, 104, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { goto __pyx_L5_argtuple_error; @@ -3066,12 +2861,12 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes values[2] = PyTuple_GET_ITEM(__pyx_args, 2); } __pyx_v_word = values[0]; - __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 113, __pyx_L3_error) - __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 113, __pyx_L3_error) + __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 104, __pyx_L3_error) + __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 104, __pyx_L3_error) } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 113, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 104, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -3090,7 +2885,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("compute_ngrams_bytes", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 113, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 104, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -5525,224 +5320,6 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { return __pyx_r; } -/* "cfunc.to_py":65 - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): - * def wrap(object b): # <<<<<<<<<<<<<< - * """wrap(b)""" - * return f(b) - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap(PyObject *__pyx_self, PyObject *__pyx_v_b); /*proto*/ -static char __pyx_doc_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap[] = "wrap(b)"; -static PyMethodDef __pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap = {"wrap", (PyCFunction)__pyx_pw_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, METH_O, __pyx_doc_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap}; -static PyObject *__pyx_pw_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap(PyObject *__pyx_self, PyObject *__pyx_v_b) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("wrap (wrapper)", 0); - __pyx_r = __pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(__pyx_self, ((PyObject *)__pyx_v_b)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(PyObject *__pyx_self, PyObject *__pyx_v_b) { - struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_cur_scope; - struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_outer_scope; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("wrap", 0); - __pyx_outer_scope = (struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *) __Pyx_CyFunction_GetClosure(__pyx_self); - __pyx_cur_scope = __pyx_outer_scope; - - /* "cfunc.to_py":67 - * def wrap(object b): - * """wrap(b)""" - * return f(b) # <<<<<<<<<<<<<< - * return wrap - * - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_cur_scope->__pyx_v_f(__pyx_v_b); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 67, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "cfunc.to_py":65 - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): - * def wrap(object b): # <<<<<<<<<<<<<< - * """wrap(b)""" - * return f(b) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("cfunc.to_py.__Pyx_CFunc_object____object___to_py.wrap", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "cfunc.to_py":64 - * - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< - * def wrap(object b): - * """wrap(b)""" - */ - -static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*__pyx_v_f)(PyObject *)) { - struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_cur_scope; - PyObject *__pyx_v_wrap = 0; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__Pyx_CFunc_object____object___to_py", 0); - __pyx_cur_scope = (struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(__pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py, __pyx_empty_tuple, NULL); - if (unlikely(!__pyx_cur_scope)) { - __pyx_cur_scope = ((struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)Py_None); - __Pyx_INCREF(Py_None); - __PYX_ERR(2, 64, __pyx_L1_error) - } else { - __Pyx_GOTREF(__pyx_cur_scope); - } - __pyx_cur_scope->__pyx_v_f = __pyx_v_f; - - /* "cfunc.to_py":65 - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): - * def wrap(object b): # <<<<<<<<<<<<<< - * """wrap(b)""" - * return f(b) - */ - __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, 0, __pyx_n_s_Pyx_CFunc_object____object___t, ((PyObject*)__pyx_cur_scope), __pyx_n_s_cfunc_to_py, __pyx_d, ((PyObject *)__pyx_codeobj__11)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 65, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_v_wrap = __pyx_t_1; - __pyx_t_1 = 0; - - /* "cfunc.to_py":68 - * """wrap(b)""" - * return f(b) - * return wrap # <<<<<<<<<<<<<< - * - * - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_wrap); - __pyx_r = __pyx_v_wrap; - goto __pyx_L0; - - /* "cfunc.to_py":64 - * - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< - * def wrap(object b): - * """wrap(b)""" - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("cfunc.to_py.__Pyx_CFunc_object____object___to_py", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_wrap); - __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[8]; -static int __pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py = 0; - -static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - PyObject *o; - if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)))) { - o = (PyObject*)__pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[--__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py]; - memset(o, 0, sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)); - (void) PyObject_INIT(o, t); - } else { - o = (*t->tp_alloc)(t, 0); - if (unlikely(!o)) return 0; - } - return o; -} - -static void __pyx_tp_dealloc___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyObject *o) { - if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)))) { - __pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py++] = ((struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)o); - } else { - (*Py_TYPE(o)->tp_free)(o); - } -} - -static PyTypeObject __pyx_scope_struct____Pyx_CFunc_object____object___to_py = { - PyVarObject_HEAD_INIT(0, 0) - "gensim.models._utils_any2vec.__pyx_scope_struct____Pyx_CFunc_object____object___to_py", /*tp_name*/ - sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc___pyx_scope_struct____Pyx_CFunc_object____object___to_py, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - static PyMethodDef __pyx_methods[] = { {"ft_hash_bytes", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash_bytes, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_ft_hash_bytes}, {"ft_hash_broken", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_2ft_hash_broken}, @@ -5801,13 +5378,9 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_MB_START, __pyx_k_MB_START, sizeof(__pyx_k_MB_START), 0, 0, 1, 1}, {&__pyx_kp_u_Non_native_byte_order_not_suppor, __pyx_k_Non_native_byte_order_not_suppor, sizeof(__pyx_k_Non_native_byte_order_not_suppor), 0, 1, 0, 0}, {&__pyx_n_s_PY2, __pyx_k_PY2, sizeof(__pyx_k_PY2), 0, 0, 1, 1}, - {&__pyx_n_s_Pyx_CFunc_object____object___t, __pyx_k_Pyx_CFunc_object____object___t, sizeof(__pyx_k_Pyx_CFunc_object____object___t), 0, 0, 1, 1}, {&__pyx_n_s_RuntimeError, __pyx_k_RuntimeError, sizeof(__pyx_k_RuntimeError), 0, 0, 1, 1}, {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, {&__pyx_kp_u__2, __pyx_k__2, sizeof(__pyx_k__2), 0, 1, 0, 0}, - {&__pyx_n_s_b, __pyx_k_b, sizeof(__pyx_k_b), 0, 0, 1, 1}, - {&__pyx_n_s_byte_to_int, __pyx_k_byte_to_int, sizeof(__pyx_k_byte_to_int), 0, 0, 1, 1}, - {&__pyx_n_s_cfunc_to_py, __pyx_k_cfunc_to_py, sizeof(__pyx_k_cfunc_to_py), 0, 0, 1, 1}, {&__pyx_n_s_cline_in_traceback, __pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 0, 1, 1}, {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, @@ -5822,15 +5395,13 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_s_numpy_core_umath_failed_to_impor, __pyx_k_numpy_core_umath_failed_to_impor, sizeof(__pyx_k_numpy_core_umath_failed_to_impor), 0, 0, 1, 0}, {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, {&__pyx_n_s_six, __pyx_k_six, sizeof(__pyx_k_six), 0, 0, 1, 1}, - {&__pyx_kp_s_stringsource, __pyx_k_stringsource, sizeof(__pyx_k_stringsource), 0, 0, 1, 0}, {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, {&__pyx_kp_u_unknown_dtype_code_in_numpy_pxd, __pyx_k_unknown_dtype_code_in_numpy_pxd, sizeof(__pyx_k_unknown_dtype_code_in_numpy_pxd), 0, 1, 0, 0}, {&__pyx_n_s_word, __pyx_k_word, sizeof(__pyx_k_word), 0, 0, 1, 1}, - {&__pyx_n_s_wrap, __pyx_k_wrap, sizeof(__pyx_k_wrap), 0, 0, 1, 1}, {0, 0, 0, 0, 0, 0, 0} }; static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(0, 100, __pyx_L1_error) + __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(0, 91, __pyx_L1_error) __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 272, __pyx_L1_error) __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_n_s_RuntimeError); if (!__pyx_builtin_RuntimeError) __PYX_ERR(1, 856, __pyx_L1_error) __pyx_builtin_ImportError = __Pyx_GetBuiltinName(__pyx_n_s_ImportError); if (!__pyx_builtin_ImportError) __PYX_ERR(1, 1038, __pyx_L1_error) @@ -5919,18 +5490,6 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __pyx_tuple__9 = PyTuple_Pack(1, __pyx_kp_s_numpy_core_umath_failed_to_impor); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(1, 1044, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__9); __Pyx_GIVEREF(__pyx_tuple__9); - - /* "cfunc.to_py":65 - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): - * def wrap(object b): # <<<<<<<<<<<<<< - * """wrap(b)""" - * return f(b) - */ - __pyx_tuple__10 = PyTuple_Pack(1, __pyx_n_s_b); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(2, 65, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__10); - __Pyx_GIVEREF(__pyx_tuple__10); - __pyx_codeobj__11 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__10, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_stringsource, __pyx_n_s_wrap, 65, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__11)) __PYX_ERR(2, 65, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -5985,17 +5544,8 @@ static int __Pyx_modinit_type_init_code(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); /*--- Type init code ---*/ - if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(2, 64, __pyx_L1_error) - __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_print = 0; - if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_dictoffset && __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_getattro == PyObject_GenericGetAttr)) { - __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; - } - __pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py = &__pyx_scope_struct____Pyx_CFunc_object____object___to_py; __Pyx_RefNannyFinishContext(); return 0; - __pyx_L1_error:; - __Pyx_RefNannyFinishContext(); - return -1; } static int __Pyx_modinit_type_import_code(void) { @@ -6003,7 +5553,7 @@ static int __Pyx_modinit_type_import_code(void) { PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); /*--- Type import code ---*/ - __pyx_t_1 = PyImport_ImportModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 9, __pyx_L1_error) + __pyx_t_1 = PyImport_ImportModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 9, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_ptype_7cpython_4type_type = __Pyx_ImportType(__pyx_t_1, __Pyx_BUILTIN_MODULE_NAME, "type", #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 @@ -6012,7 +5562,7 @@ static int __Pyx_modinit_type_import_code(void) { sizeof(PyHeapTypeObject), #endif __Pyx_ImportType_CheckSize_Warn); - if (!__pyx_ptype_7cpython_4type_type) __PYX_ERR(3, 9, __pyx_L1_error) + if (!__pyx_ptype_7cpython_4type_type) __PYX_ERR(2, 9, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = PyImport_ImportModule("numpy"); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 206, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); @@ -6145,7 +5695,6 @@ static CYTHON_SMALL_CODE int __pyx_pymod_exec__utils_any2vec(PyObject *__pyx_pyi { PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; - int __pyx_t_3; __Pyx_RefNannyDeclarations #if CYTHON_PEP489_MULTI_PHASE_INIT if (__pyx_m) { @@ -6242,7 +5791,7 @@ if (!__Pyx_RefNanny) { (void)__Pyx_modinit_global_init_code(); (void)__Pyx_modinit_variable_export_code(); (void)__Pyx_modinit_function_export_code(); - if (unlikely(__Pyx_modinit_type_init_code() != 0)) goto __pyx_L1_error; + (void)__Pyx_modinit_type_init_code(); if (unlikely(__Pyx_modinit_type_import_code() != 0)) goto __pyx_L1_error; (void)__Pyx_modinit_variable_import_code(); (void)__Pyx_modinit_function_import_code(); @@ -6284,48 +5833,23 @@ if (!__Pyx_RefNanny) { if (PyDict_SetItem(__pyx_d, __pyx_n_s_np, __pyx_t_2) < 0) __PYX_ERR(0, 18, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "gensim/models/_utils_any2vec.pyx":28 - * return ord(b) + /* "gensim/models/_utils_any2vec.pyx":100 + * # as opposed to a new character. + * # + * _MB_MASK = 0xC0 # <<<<<<<<<<<<<< + * _MB_START = 0x80 * - * _byte_to_int = _byte_to_int_py2 if PY2 else _byte_to_int_py3 # <<<<<<<<<<<<<< + */ + if (PyDict_SetItem(__pyx_d, __pyx_n_s_MB_MASK, __pyx_int_192) < 0) __PYX_ERR(0, 100, __pyx_L1_error) + + /* "gensim/models/_utils_any2vec.pyx":101 + * # + * _MB_MASK = 0xC0 + * _MB_START = 0x80 # <<<<<<<<<<<<<< * * */ - __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 28, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 28, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (__pyx_t_3) { - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 28, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_t_1; - __pyx_t_1 = 0; - } else { - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__byte_to_int_py3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 28, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_t_1; - __pyx_t_1 = 0; - } - if (PyDict_SetItem(__pyx_d, __pyx_n_s_byte_to_int, __pyx_t_2) < 0) __PYX_ERR(0, 28, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "gensim/models/_utils_any2vec.pyx":109 - * # as opposed to a new character. - * # - * _MB_MASK = 0xC0 # <<<<<<<<<<<<<< - * _MB_START = 0x80 - * - */ - if (PyDict_SetItem(__pyx_d, __pyx_n_s_MB_MASK, __pyx_int_192) < 0) __PYX_ERR(0, 109, __pyx_L1_error) - - /* "gensim/models/_utils_any2vec.pyx":110 - * # - * _MB_MASK = 0xC0 - * _MB_START = 0x80 # <<<<<<<<<<<<<< - * - * - */ - if (PyDict_SetItem(__pyx_d, __pyx_n_s_MB_START, __pyx_int_128) < 0) __PYX_ERR(0, 110, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_MB_START, __pyx_int_128) < 0) __PYX_ERR(0, 101, __pyx_L1_error) /* "gensim/models/_utils_any2vec.pyx":1 * #!/usr/bin/env cython # <<<<<<<<<<<<<< @@ -6337,12 +5861,12 @@ if (!__Pyx_RefNanny) { if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_2) < 0) __PYX_ERR(0, 1, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "cfunc.to_py":64 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 + * raise ImportError("numpy.core.umath failed to import") * - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< - * def wrap(object b): - * """wrap(b)""" + * cdef inline int import_ufunc() except -1: # <<<<<<<<<<<<<< + * try: + * _import_umath() */ /*--- Wrapped vars code ---*/ @@ -6416,6 +5940,43 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name) { return result; } +/* ArgTypeTest */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) +{ + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + else if (exact) { + #if PY_MAJOR_VERSION == 2 + if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; + #endif + } + else { + if (likely(__Pyx_TypeCheck(obj, type))) return 1; + } + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", + name, type->tp_name, Py_TYPE(obj)->tp_name); + return 0; +} + +/* unicode_iter */ +static CYTHON_INLINE int __Pyx_init_unicode_iteration( + PyObject* ustring, Py_ssize_t *length, void** data, int *kind) { +#if CYTHON_PEP393_ENABLED + if (unlikely(__Pyx_PyUnicode_READY(ustring) < 0)) return -1; + *kind = PyUnicode_KIND(ustring); + *length = PyUnicode_GET_LENGTH(ustring); + *data = PyUnicode_DATA(ustring); +#else + *kind = 0; + *length = PyUnicode_GET_SIZE(ustring); + *data = (void*)PyUnicode_AS_UNICODE(ustring); +#endif + return 0; +} + /* UnicodeAsUCS4 */ static CYTHON_INLINE Py_UCS4 __Pyx_PyUnicode_AsPy_UCS4(PyObject* x) { Py_ssize_t length; @@ -6476,43 +6037,6 @@ static long __Pyx__PyObject_Ord(PyObject* c) { return (long)(Py_UCS4)-1; } -/* ArgTypeTest */ -static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) -{ - if (unlikely(!type)) { - PyErr_SetString(PyExc_SystemError, "Missing type object"); - return 0; - } - else if (exact) { - #if PY_MAJOR_VERSION == 2 - if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; - #endif - } - else { - if (likely(__Pyx_TypeCheck(obj, type))) return 1; - } - PyErr_Format(PyExc_TypeError, - "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", - name, type->tp_name, Py_TYPE(obj)->tp_name); - return 0; -} - -/* unicode_iter */ -static CYTHON_INLINE int __Pyx_init_unicode_iteration( - PyObject* ustring, Py_ssize_t *length, void** data, int *kind) { -#if CYTHON_PEP393_ENABLED - if (unlikely(__Pyx_PyUnicode_READY(ustring) < 0)) return -1; - *kind = PyUnicode_KIND(ustring); - *length = PyUnicode_GET_LENGTH(ustring); - *data = PyUnicode_DATA(ustring); -#else - *kind = 0; - *length = PyUnicode_GET_SIZE(ustring); - *data = (void*)PyUnicode_AS_UNICODE(ustring); -#endif - return 0; -} - /* JoinPyUnicode */ static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_count, Py_ssize_t result_ulength, CYTHON_UNUSED Py_UCS4 max_char) { @@ -7602,681 +7126,6 @@ static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) return -1; } -/* FetchCommonType */ -static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { - PyObject* fake_module; - PyTypeObject* cached_type = NULL; - fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); - if (!fake_module) return NULL; - Py_INCREF(fake_module); - cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); - if (cached_type) { - if (!PyType_Check((PyObject*)cached_type)) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s is not a type object", - type->tp_name); - goto bad; - } - if (cached_type->tp_basicsize != type->tp_basicsize) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s has the wrong size, try recompiling", - type->tp_name); - goto bad; - } - } else { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; - PyErr_Clear(); - if (PyType_Ready(type) < 0) goto bad; - if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) - goto bad; - Py_INCREF(type); - cached_type = type; - } -done: - Py_DECREF(fake_module); - return cached_type; -bad: - Py_XDECREF(cached_type); - cached_type = NULL; - goto done; -} - -/* CythonFunction */ -#include -static PyObject * -__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure) -{ - if (unlikely(op->func_doc == NULL)) { - if (op->func.m_ml->ml_doc) { -#if PY_MAJOR_VERSION >= 3 - op->func_doc = PyUnicode_FromString(op->func.m_ml->ml_doc); -#else - op->func_doc = PyString_FromString(op->func.m_ml->ml_doc); -#endif - if (unlikely(op->func_doc == NULL)) - return NULL; - } else { - Py_INCREF(Py_None); - return Py_None; - } - } - Py_INCREF(op->func_doc); - return op->func_doc; -} -static int -__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) -{ - PyObject *tmp = op->func_doc; - if (value == NULL) { - value = Py_None; - } - Py_INCREF(value); - op->func_doc = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - if (unlikely(op->func_name == NULL)) { -#if PY_MAJOR_VERSION >= 3 - op->func_name = PyUnicode_InternFromString(op->func.m_ml->ml_name); -#else - op->func_name = PyString_InternFromString(op->func.m_ml->ml_name); -#endif - if (unlikely(op->func_name == NULL)) - return NULL; - } - Py_INCREF(op->func_name); - return op->func_name; -} -static int -__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) -{ - PyObject *tmp; -#if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) -#else - if (unlikely(value == NULL || !PyString_Check(value))) -#endif - { - PyErr_SetString(PyExc_TypeError, - "__name__ must be set to a string object"); - return -1; - } - tmp = op->func_name; - Py_INCREF(value); - op->func_name = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - Py_INCREF(op->func_qualname); - return op->func_qualname; -} -static int -__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) -{ - PyObject *tmp; -#if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) -#else - if (unlikely(value == NULL || !PyString_Check(value))) -#endif - { - PyErr_SetString(PyExc_TypeError, - "__qualname__ must be set to a string object"); - return -1; - } - tmp = op->func_qualname; - Py_INCREF(value); - op->func_qualname = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_self(__pyx_CyFunctionObject *m, CYTHON_UNUSED void *closure) -{ - PyObject *self; - self = m->func_closure; - if (self == NULL) - self = Py_None; - Py_INCREF(self); - return self; -} -static PyObject * -__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - if (unlikely(op->func_dict == NULL)) { - op->func_dict = PyDict_New(); - if (unlikely(op->func_dict == NULL)) - return NULL; - } - Py_INCREF(op->func_dict); - return op->func_dict; -} -static int -__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) -{ - PyObject *tmp; - if (unlikely(value == NULL)) { - PyErr_SetString(PyExc_TypeError, - "function's dictionary may not be deleted"); - return -1; - } - if (unlikely(!PyDict_Check(value))) { - PyErr_SetString(PyExc_TypeError, - "setting function's dictionary to a non-dict"); - return -1; - } - tmp = op->func_dict; - Py_INCREF(value); - op->func_dict = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - Py_INCREF(op->func_globals); - return op->func_globals; -} -static PyObject * -__Pyx_CyFunction_get_closure(CYTHON_UNUSED __pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - Py_INCREF(Py_None); - return Py_None; -} -static PyObject * -__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - PyObject* result = (op->func_code) ? op->func_code : Py_None; - Py_INCREF(result); - return result; -} -static int -__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { - int result = 0; - PyObject *res = op->defaults_getter((PyObject *) op); - if (unlikely(!res)) - return -1; - #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - op->defaults_tuple = PyTuple_GET_ITEM(res, 0); - Py_INCREF(op->defaults_tuple); - op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); - Py_INCREF(op->defaults_kwdict); - #else - op->defaults_tuple = PySequence_ITEM(res, 0); - if (unlikely(!op->defaults_tuple)) result = -1; - else { - op->defaults_kwdict = PySequence_ITEM(res, 1); - if (unlikely(!op->defaults_kwdict)) result = -1; - } - #endif - Py_DECREF(res); - return result; -} -static int -__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; - if (!value) { - value = Py_None; - } else if (value != Py_None && !PyTuple_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "__defaults__ must be set to a tuple object"); - return -1; - } - Py_INCREF(value); - tmp = op->defaults_tuple; - op->defaults_tuple = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { - PyObject* result = op->defaults_tuple; - if (unlikely(!result)) { - if (op->defaults_getter) { - if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; - result = op->defaults_tuple; - } else { - result = Py_None; - } - } - Py_INCREF(result); - return result; -} -static int -__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; - if (!value) { - value = Py_None; - } else if (value != Py_None && !PyDict_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "__kwdefaults__ must be set to a dict object"); - return -1; - } - Py_INCREF(value); - tmp = op->defaults_kwdict; - op->defaults_kwdict = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { - PyObject* result = op->defaults_kwdict; - if (unlikely(!result)) { - if (op->defaults_getter) { - if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; - result = op->defaults_kwdict; - } else { - result = Py_None; - } - } - Py_INCREF(result); - return result; -} -static int -__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; - if (!value || value == Py_None) { - value = NULL; - } else if (!PyDict_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "__annotations__ must be set to a dict object"); - return -1; - } - Py_XINCREF(value); - tmp = op->func_annotations; - op->func_annotations = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { - PyObject* result = op->func_annotations; - if (unlikely(!result)) { - result = PyDict_New(); - if (unlikely(!result)) return NULL; - op->func_annotations = result; - } - Py_INCREF(result); - return result; -} -static PyGetSetDef __pyx_CyFunction_getsets[] = { - {(char *) "func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, - {(char *) "__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, - {(char *) "func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, - {(char *) "__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, - {(char *) "__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, - {(char *) "__self__", (getter)__Pyx_CyFunction_get_self, 0, 0, 0}, - {(char *) "func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, - {(char *) "__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, - {(char *) "func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, - {(char *) "__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, - {(char *) "func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, - {(char *) "__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, - {(char *) "func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, - {(char *) "__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, - {(char *) "func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, - {(char *) "__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, - {(char *) "__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, - {(char *) "__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, - {0, 0, 0, 0, 0} -}; -static PyMemberDef __pyx_CyFunction_members[] = { - {(char *) "__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), PY_WRITE_RESTRICTED, 0}, - {0, 0, 0, 0, 0} -}; -static PyObject * -__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, CYTHON_UNUSED PyObject *args) -{ -#if PY_MAJOR_VERSION >= 3 - return PyUnicode_FromString(m->func.m_ml->ml_name); -#else - return PyString_FromString(m->func.m_ml->ml_name); -#endif -} -static PyMethodDef __pyx_CyFunction_methods[] = { - {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, - {0, 0, 0, 0} -}; -#if PY_VERSION_HEX < 0x030500A0 -#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) -#else -#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func.m_weakreflist) -#endif -static PyObject *__Pyx_CyFunction_New(PyTypeObject *type, PyMethodDef *ml, int flags, PyObject* qualname, - PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { - __pyx_CyFunctionObject *op = PyObject_GC_New(__pyx_CyFunctionObject, type); - if (op == NULL) - return NULL; - op->flags = flags; - __Pyx_CyFunction_weakreflist(op) = NULL; - op->func.m_ml = ml; - op->func.m_self = (PyObject *) op; - Py_XINCREF(closure); - op->func_closure = closure; - Py_XINCREF(module); - op->func.m_module = module; - op->func_dict = NULL; - op->func_name = NULL; - Py_INCREF(qualname); - op->func_qualname = qualname; - op->func_doc = NULL; - op->func_classobj = NULL; - op->func_globals = globals; - Py_INCREF(op->func_globals); - Py_XINCREF(code); - op->func_code = code; - op->defaults_pyobjects = 0; - op->defaults = NULL; - op->defaults_tuple = NULL; - op->defaults_kwdict = NULL; - op->defaults_getter = NULL; - op->func_annotations = NULL; - PyObject_GC_Track(op); - return (PyObject *) op; -} -static int -__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) -{ - Py_CLEAR(m->func_closure); - Py_CLEAR(m->func.m_module); - Py_CLEAR(m->func_dict); - Py_CLEAR(m->func_name); - Py_CLEAR(m->func_qualname); - Py_CLEAR(m->func_doc); - Py_CLEAR(m->func_globals); - Py_CLEAR(m->func_code); - Py_CLEAR(m->func_classobj); - Py_CLEAR(m->defaults_tuple); - Py_CLEAR(m->defaults_kwdict); - Py_CLEAR(m->func_annotations); - if (m->defaults) { - PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); - int i; - for (i = 0; i < m->defaults_pyobjects; i++) - Py_XDECREF(pydefaults[i]); - PyObject_Free(m->defaults); - m->defaults = NULL; - } - return 0; -} -static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) -{ - if (__Pyx_CyFunction_weakreflist(m) != NULL) - PyObject_ClearWeakRefs((PyObject *) m); - __Pyx_CyFunction_clear(m); - PyObject_GC_Del(m); -} -static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) -{ - PyObject_GC_UnTrack(m); - __Pyx__CyFunction_dealloc(m); -} -static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) -{ - Py_VISIT(m->func_closure); - Py_VISIT(m->func.m_module); - Py_VISIT(m->func_dict); - Py_VISIT(m->func_name); - Py_VISIT(m->func_qualname); - Py_VISIT(m->func_doc); - Py_VISIT(m->func_globals); - Py_VISIT(m->func_code); - Py_VISIT(m->func_classobj); - Py_VISIT(m->defaults_tuple); - Py_VISIT(m->defaults_kwdict); - if (m->defaults) { - PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); - int i; - for (i = 0; i < m->defaults_pyobjects; i++) - Py_VISIT(pydefaults[i]); - } - return 0; -} -static PyObject *__Pyx_CyFunction_descr_get(PyObject *func, PyObject *obj, PyObject *type) -{ - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - if (m->flags & __Pyx_CYFUNCTION_STATICMETHOD) { - Py_INCREF(func); - return func; - } - if (m->flags & __Pyx_CYFUNCTION_CLASSMETHOD) { - if (type == NULL) - type = (PyObject *)(Py_TYPE(obj)); - return __Pyx_PyMethod_New(func, type, (PyObject *)(Py_TYPE(type))); - } - if (obj == Py_None) - obj = NULL; - return __Pyx_PyMethod_New(func, obj, type); -} -static PyObject* -__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) -{ -#if PY_MAJOR_VERSION >= 3 - return PyUnicode_FromFormat("", - op->func_qualname, (void *)op); -#else - return PyString_FromFormat("", - PyString_AsString(op->func_qualname), (void *)op); -#endif -} -static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { - PyCFunctionObject* f = (PyCFunctionObject*)func; - PyCFunction meth = f->m_ml->ml_meth; - Py_ssize_t size; - switch (f->m_ml->ml_flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { - case METH_VARARGS: - if (likely(kw == NULL || PyDict_Size(kw) == 0)) - return (*meth)(self, arg); - break; - case METH_VARARGS | METH_KEYWORDS: - return (*(PyCFunctionWithKeywords)(void*)meth)(self, arg, kw); - case METH_NOARGS: - if (likely(kw == NULL || PyDict_Size(kw) == 0)) { - size = PyTuple_GET_SIZE(arg); - if (likely(size == 0)) - return (*meth)(self, NULL); - PyErr_Format(PyExc_TypeError, - "%.200s() takes no arguments (%" CYTHON_FORMAT_SSIZE_T "d given)", - f->m_ml->ml_name, size); - return NULL; - } - break; - case METH_O: - if (likely(kw == NULL || PyDict_Size(kw) == 0)) { - size = PyTuple_GET_SIZE(arg); - if (likely(size == 1)) { - PyObject *result, *arg0; - #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - arg0 = PyTuple_GET_ITEM(arg, 0); - #else - arg0 = PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; - #endif - result = (*meth)(self, arg0); - #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) - Py_DECREF(arg0); - #endif - return result; - } - PyErr_Format(PyExc_TypeError, - "%.200s() takes exactly one argument (%" CYTHON_FORMAT_SSIZE_T "d given)", - f->m_ml->ml_name, size); - return NULL; - } - break; - default: - PyErr_SetString(PyExc_SystemError, "Bad call flags in " - "__Pyx_CyFunction_Call. METH_OLDARGS is no " - "longer supported!"); - return NULL; - } - PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", - f->m_ml->ml_name); - return NULL; -} -static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { - return __Pyx_CyFunction_CallMethod(func, ((PyCFunctionObject*)func)->m_self, arg, kw); -} -static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { - PyObject *result; - __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; - if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { - Py_ssize_t argc; - PyObject *new_args; - PyObject *self; - argc = PyTuple_GET_SIZE(args); - new_args = PyTuple_GetSlice(args, 1, argc); - if (unlikely(!new_args)) - return NULL; - self = PyTuple_GetItem(args, 0); - if (unlikely(!self)) { - Py_DECREF(new_args); - return NULL; - } - result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); - Py_DECREF(new_args); - } else { - result = __Pyx_CyFunction_Call(func, args, kw); - } - return result; -} -static PyTypeObject __pyx_CyFunctionType_type = { - PyVarObject_HEAD_INIT(0, 0) - "cython_function_or_method", - sizeof(__pyx_CyFunctionObject), - 0, - (destructor) __Pyx_CyFunction_dealloc, - 0, - 0, - 0, -#if PY_MAJOR_VERSION < 3 - 0, -#else - 0, -#endif - (reprfunc) __Pyx_CyFunction_repr, - 0, - 0, - 0, - 0, - __Pyx_CyFunction_CallAsMethod, - 0, - 0, - 0, - 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - 0, - (traverseproc) __Pyx_CyFunction_traverse, - (inquiry) __Pyx_CyFunction_clear, - 0, -#if PY_VERSION_HEX < 0x030500A0 - offsetof(__pyx_CyFunctionObject, func_weakreflist), -#else - offsetof(PyCFunctionObject, m_weakreflist), -#endif - 0, - 0, - __pyx_CyFunction_methods, - __pyx_CyFunction_members, - __pyx_CyFunction_getsets, - 0, - 0, - __Pyx_CyFunction_descr_get, - 0, - offsetof(__pyx_CyFunctionObject, func_dict), - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, -#if PY_VERSION_HEX >= 0x030400a1 - 0, -#endif -}; -static int __pyx_CyFunction_init(void) { - __pyx_CyFunctionType = __Pyx_FetchCommonType(&__pyx_CyFunctionType_type); - if (unlikely(__pyx_CyFunctionType == NULL)) { - return -1; - } - return 0; -} -static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *func, size_t size, int pyobjects) { - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - m->defaults = PyObject_Malloc(size); - if (unlikely(!m->defaults)) - return PyErr_NoMemory(); - memset(m->defaults, 0, size); - m->defaults_pyobjects = pyobjects; - return m->defaults; -} -static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - m->defaults_tuple = tuple; - Py_INCREF(tuple); -} -static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - m->defaults_kwdict = dict; - Py_INCREF(dict); -} -static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - m->func_annotations = dict; - Py_INCREF(dict); -} - -/* PyObject_GenericGetAttrNoDict */ -#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 -static PyObject *__Pyx_RaiseGenericGetAttributeError(PyTypeObject *tp, PyObject *attr_name) { - PyErr_Format(PyExc_AttributeError, -#if PY_MAJOR_VERSION >= 3 - "'%.50s' object has no attribute '%U'", - tp->tp_name, attr_name); -#else - "'%.50s' object has no attribute '%.400s'", - tp->tp_name, PyString_AS_STRING(attr_name)); -#endif - return NULL; -} -static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name) { - PyObject *descr; - PyTypeObject *tp = Py_TYPE(obj); - if (unlikely(!PyString_Check(attr_name))) { - return PyObject_GenericGetAttr(obj, attr_name); - } - assert(!tp->tp_dictoffset); - descr = _PyType_Lookup(tp, attr_name); - if (unlikely(!descr)) { - return __Pyx_RaiseGenericGetAttributeError(tp, attr_name); - } - Py_INCREF(descr); - #if PY_MAJOR_VERSION < 3 - if (likely(PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_HAVE_CLASS))) - #endif - { - descrgetfunc f = Py_TYPE(descr)->tp_descr_get; - if (unlikely(f)) { - PyObject *res = f(descr, obj, (PyObject *)tp); - Py_DECREF(descr); - return res; - } - } - return descr; -} -#endif - /* TypeImport */ #ifndef __PYX_HAVE_RT_ImportType #define __PYX_HAVE_RT_ImportType @@ -8646,37 +7495,6 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, return (target_type) value;\ } -/* CIntToPy */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { - const long neg_one = (long) ((long) 0 - (long) 1), const_zero = (long) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(long) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(long) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(long) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(long), - little, !is_unsigned); - } -} - /* CIntToPy */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint32_t(uint32_t value) { const uint32_t neg_one = (uint32_t) ((uint32_t) 0 - (uint32_t) 1), const_zero = (uint32_t) 0; @@ -9678,6 +8496,37 @@ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { return (int) -1; } +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { + const long neg_one = (long) ((long) 0 - (long) 1), const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); + } +} + /* CIntFromPy */ static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { const long neg_one = (long) ((long) 0 - (long) 1), const_zero = (long) 0; diff --git a/gensim/models/_utils_any2vec.pyx b/gensim/models/_utils_any2vec.pyx index cecd3b0707..94622ad567 100644 --- a/gensim/models/_utils_any2vec.pyx +++ b/gensim/models/_utils_any2vec.pyx @@ -131,19 +131,16 @@ cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): ngrams = [] for i in range(num_bytes): - ngram = [] - if utf8_word[i] & _MB_MASK == _MB_START: continue j, n = i, 1 while j < num_bytes and n <= max_n: - ngram.append(utf8_word[j]) j += 1 while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: - ngram.append(utf8_word[j]) j += 1 if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + ngram = bytes(utf8_word[i:j]) ngrams.append(bytes(ngram)) n += 1 return ngrams diff --git a/gensim/models/utils_any2vec.py b/gensim/models/utils_any2vec.py index 9083fa8064..ff697c290a 100644 --- a/gensim/models/utils_any2vec.py +++ b/gensim/models/utils_any2vec.py @@ -167,20 +167,17 @@ def _compute_ngrams_bytes_py(word, min_n, max_n): ngrams = [] for i in range(num_bytes): - ngram = [] - if utf8_word[i] & _MB_MASK == _MB_START: continue j, n = i, 1 while j < num_bytes and n <= max_n: - ngram.append(utf8_word[j]) j += 1 while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: - ngram.append(utf8_word[j]) j += 1 if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): - ngrams.append(bytes(ngram)) + ngram = bytes(utf8_word[i:j]) + ngrams.append(ngram) n += 1 return ngrams From cdab5b6949ccbc6120c8f5ebd7d1d85f8c65a6c6 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Wed, 23 Jan 2019 13:16:58 +1100 Subject: [PATCH 21/32] avoid using fstrings for Py2 compatibility --- gensim/models/_utils_any2vec.c | 546 ++++++++++++++++--------------- gensim/models/_utils_any2vec.pyx | 2 +- gensim/models/utils_any2vec.py | 2 +- 3 files changed, 293 insertions(+), 257 deletions(-) diff --git a/gensim/models/_utils_any2vec.c b/gensim/models/_utils_any2vec.c index 6e7118f529..d4d5caeb09 100644 --- a/gensim/models/_utils_any2vec.c +++ b/gensim/models/_utils_any2vec.c @@ -1271,6 +1271,47 @@ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ const char* function_name); +/* PyCFunctionFastCall.proto */ +#if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); +#else +#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) +#endif + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); +#else +#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) +#endif +#define __Pyx_BUILD_ASSERT_EXPR(cond)\ + (sizeof(char [1 - 2*!(cond)]) - 1) +#ifndef Py_MEMBER_SIZE +#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) +#endif + static size_t __pyx_pyframe_localsplus_offset = 0; + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets()\ + ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\ + (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) + #define __Pyx_PyFrame_GetLocalsplus(frame)\ + (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) +#endif + +/* PyObjectCall2Args.proto */ +static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectCallOneArg.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + /* GetItemInt.proto */ #define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ @@ -1314,44 +1355,6 @@ static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_ve static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); #endif -/* PyCFunctionFastCall.proto */ -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); -#else -#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) -#endif - -/* PyFunctionFastCall.proto */ -#if CYTHON_FAST_PYCALL -#define __Pyx_PyFunction_FastCall(func, args, nargs)\ - __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); -#else -#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) -#endif -#define __Pyx_BUILD_ASSERT_EXPR(cond)\ - (sizeof(char [1 - 2*!(cond)]) - 1) -#ifndef Py_MEMBER_SIZE -#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) -#endif - static size_t __pyx_pyframe_localsplus_offset = 0; - #include "frameobject.h" - #define __Pxy_PyFrame_Initialize_Offsets()\ - ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\ - (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) - #define __Pyx_PyFrame_GetLocalsplus(frame)\ - (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) -#endif - -/* PyObjectCallMethO.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); -#endif - -/* PyObjectCallOneArg.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); - /* PyThreadStateGet.proto */ #if CYTHON_FAST_THREAD_STATE #define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; @@ -1678,6 +1681,7 @@ static PyObject *__pyx_builtin_ValueError; static PyObject *__pyx_builtin_RuntimeError; static PyObject *__pyx_builtin_ImportError; static const char __pyx_k_[] = "<"; +static const char __pyx_k_s[] = "<%s>"; static const char __pyx_k__2[] = ">"; static const char __pyx_k_np[] = "np"; static const char __pyx_k_PY2[] = "PY2"; @@ -1690,6 +1694,8 @@ static const char __pyx_k_max_n[] = "max_n"; static const char __pyx_k_min_n[] = "min_n"; static const char __pyx_k_numpy[] = "numpy"; static const char __pyx_k_range[] = "range"; +static const char __pyx_k_utf_8[] = "utf-8"; +static const char __pyx_k_encode[] = "encode"; static const char __pyx_k_import[] = "__import__"; static const char __pyx_k_MB_MASK[] = "_MB_MASK"; static const char __pyx_k_MB_START[] = "_MB_START"; @@ -1718,6 +1724,7 @@ static PyObject *__pyx_n_s_RuntimeError; static PyObject *__pyx_n_s_ValueError; static PyObject *__pyx_kp_u__2; static PyObject *__pyx_n_s_cline_in_traceback; +static PyObject *__pyx_n_s_encode; static PyObject *__pyx_n_s_import; static PyObject *__pyx_n_s_main; static PyObject *__pyx_n_s_max_n; @@ -1730,9 +1737,11 @@ static PyObject *__pyx_n_s_numpy; static PyObject *__pyx_kp_s_numpy_core_multiarray_failed_to; static PyObject *__pyx_kp_s_numpy_core_umath_failed_to_impor; static PyObject *__pyx_n_s_range; +static PyObject *__pyx_kp_s_s; static PyObject *__pyx_n_s_six; static PyObject *__pyx_n_s_test; static PyObject *__pyx_kp_u_unknown_dtype_code_in_numpy_pxd; +static PyObject *__pyx_kp_s_utf_8; static PyObject *__pyx_n_s_word; static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_bytez); /* proto */ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string); /* proto */ @@ -2475,59 +2484,54 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; - Py_ssize_t __pyx_t_2; - Py_UCS4 __pyx_t_3; - PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + Py_ssize_t __pyx_t_4; int __pyx_t_5; int __pyx_t_6; int __pyx_t_7; - PyObject *__pyx_t_8 = NULL; + int __pyx_t_8; int __pyx_t_9; int __pyx_t_10; int __pyx_t_11; int __pyx_t_12; int __pyx_t_13; - int __pyx_t_14; __Pyx_RefNannySetupContext("compute_ngrams_bytes", 0); /* "gensim/models/_utils_any2vec.pyx":128 * * """ - * cdef bytes utf8_word = f'<{word}>'.encode("utf-8") # <<<<<<<<<<<<<< + * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") # <<<<<<<<<<<<<< * cdef int num_bytes = len(utf8_word) * cdef int j, i, n */ - __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 128, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = 0; - __pyx_t_3 = 127; - __Pyx_INCREF(__pyx_kp_u_); - __pyx_t_2 += 1; - __Pyx_GIVEREF(__pyx_kp_u_); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_kp_u_); - __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_word, __pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 128, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_3 = (__Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) > __pyx_t_3) ? __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) : __pyx_t_3; - __pyx_t_2 += __Pyx_PyUnicode_GET_LENGTH(__pyx_t_4); - __Pyx_GIVEREF(__pyx_t_4); - PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_4); - __pyx_t_4 = 0; - __Pyx_INCREF(__pyx_kp_u__2); - __pyx_t_2 += 1; - __Pyx_GIVEREF(__pyx_kp_u__2); - PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_kp_u__2); - __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_1, 3, __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 128, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyUnicode_AsUTF8String(((PyObject*)__pyx_t_4)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 128, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyString_FormatSafe(__pyx_kp_s_s, __pyx_v_word); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 128, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 128, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_2)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + } + } + __pyx_t_1 = (__pyx_t_2) ? __Pyx_PyObject_Call2Args(__pyx_t_3, __pyx_t_2, __pyx_kp_s_utf_8) : __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_kp_s_utf_8); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 128, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 128, __pyx_L1_error) __pyx_v_utf8_word = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; /* "gensim/models/_utils_any2vec.pyx":129 * """ - * cdef bytes utf8_word = f'<{word}>'.encode("utf-8") + * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") * cdef int num_bytes = len(utf8_word) # <<<<<<<<<<<<<< * cdef int j, i, n * @@ -2536,8 +2540,8 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); __PYX_ERR(0, 129, __pyx_L1_error) } - __pyx_t_2 = PyBytes_GET_SIZE(__pyx_v_utf8_word); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 129, __pyx_L1_error) - __pyx_v_num_bytes = __pyx_t_2; + __pyx_t_4 = PyBytes_GET_SIZE(__pyx_v_utf8_word); if (unlikely(__pyx_t_4 == ((Py_ssize_t)-1))) __PYX_ERR(0, 129, __pyx_L1_error) + __pyx_v_num_bytes = __pyx_t_4; /* "gensim/models/_utils_any2vec.pyx":132 * cdef int j, i, n @@ -2572,20 +2576,20 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P */ __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 134, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 134, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_8 = PyNumber_And(__pyx_t_1, __pyx_t_4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 134, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_8); + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 134, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = PyNumber_And(__pyx_t_1, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 134, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 134, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_1 = PyObject_RichCompare(__pyx_t_8, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 134, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_9 < 0)) __PYX_ERR(0, 134, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 134, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_1 = PyObject_RichCompare(__pyx_t_2, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 134, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_8 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_8 < 0)) __PYX_ERR(0, 134, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (__pyx_t_9) { + if (__pyx_t_8) { /* "gensim/models/_utils_any2vec.pyx":135 * for i in range(num_bytes): @@ -2612,10 +2616,10 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P * while j < num_bytes and n <= max_n: * j += 1 */ - __pyx_t_10 = __pyx_v_i; - __pyx_t_11 = 1; - __pyx_v_j = __pyx_t_10; - __pyx_v_n = __pyx_t_11; + __pyx_t_9 = __pyx_v_i; + __pyx_t_10 = 1; + __pyx_v_j = __pyx_t_9; + __pyx_v_n = __pyx_t_10; /* "gensim/models/_utils_any2vec.pyx":138 * @@ -2625,16 +2629,16 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: */ while (1) { - __pyx_t_12 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); - if (__pyx_t_12) { + __pyx_t_11 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); + if (__pyx_t_11) { } else { - __pyx_t_9 = __pyx_t_12; + __pyx_t_8 = __pyx_t_11; goto __pyx_L8_bool_binop_done; } - __pyx_t_12 = ((__pyx_v_n <= __pyx_v_max_n) != 0); - __pyx_t_9 = __pyx_t_12; + __pyx_t_11 = ((__pyx_v_n <= __pyx_v_max_n) != 0); + __pyx_t_8 = __pyx_t_11; __pyx_L8_bool_binop_done:; - if (!__pyx_t_9) break; + if (!__pyx_t_8) break; /* "gensim/models/_utils_any2vec.pyx":139 * j, n = i, 1 @@ -2653,30 +2657,30 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): */ while (1) { - __pyx_t_12 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); - if (__pyx_t_12) { + __pyx_t_11 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); + if (__pyx_t_11) { } else { - __pyx_t_9 = __pyx_t_12; + __pyx_t_8 = __pyx_t_11; goto __pyx_L12_bool_binop_done; } __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 140, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 140, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_8 = PyNumber_And(__pyx_t_1, __pyx_t_4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 140, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_8); + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 140, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = PyNumber_And(__pyx_t_1, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 140, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 140, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_1 = PyObject_RichCompare(__pyx_t_8, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 140, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_12 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_12 < 0)) __PYX_ERR(0, 140, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 140, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_1 = PyObject_RichCompare(__pyx_t_2, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 140, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_11 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_11 < 0)) __PYX_ERR(0, 140, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_9 = __pyx_t_12; + __pyx_t_8 = __pyx_t_11; __pyx_L12_bool_binop_done:; - if (!__pyx_t_9) break; + if (!__pyx_t_8) break; /* "gensim/models/_utils_any2vec.pyx":141 * j += 1 @@ -2695,31 +2699,31 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P * ngram = bytes(utf8_word[i:j]) * ngrams.append(bytes(ngram)) */ - __pyx_t_12 = ((__pyx_v_n >= __pyx_v_min_n) != 0); - if (__pyx_t_12) { + __pyx_t_11 = ((__pyx_v_n >= __pyx_v_min_n) != 0); + if (__pyx_t_11) { } else { - __pyx_t_9 = __pyx_t_12; + __pyx_t_8 = __pyx_t_11; goto __pyx_L15_bool_binop_done; } - __pyx_t_13 = ((__pyx_v_n == 1) != 0); - if (__pyx_t_13) { + __pyx_t_12 = ((__pyx_v_n == 1) != 0); + if (__pyx_t_12) { } else { - __pyx_t_12 = __pyx_t_13; + __pyx_t_11 = __pyx_t_12; goto __pyx_L17_bool_binop_done; } - __pyx_t_13 = ((__pyx_v_i == 0) != 0); - if (!__pyx_t_13) { + __pyx_t_12 = ((__pyx_v_i == 0) != 0); + if (!__pyx_t_12) { } else { - __pyx_t_12 = __pyx_t_13; + __pyx_t_11 = __pyx_t_12; goto __pyx_L17_bool_binop_done; } - __pyx_t_13 = ((__pyx_v_j == __pyx_v_num_bytes) != 0); - __pyx_t_12 = __pyx_t_13; + __pyx_t_12 = ((__pyx_v_j == __pyx_v_num_bytes) != 0); + __pyx_t_11 = __pyx_t_12; __pyx_L17_bool_binop_done:; - __pyx_t_13 = ((!__pyx_t_12) != 0); - __pyx_t_9 = __pyx_t_13; + __pyx_t_12 = ((!__pyx_t_11) != 0); + __pyx_t_8 = __pyx_t_12; __pyx_L15_bool_binop_done:; - if (__pyx_t_9) { + if (__pyx_t_8) { /* "gensim/models/_utils_any2vec.pyx":143 * j += 1 @@ -2734,11 +2738,11 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P } __pyx_t_1 = PySequence_GetSlice(__pyx_v_utf8_word, __pyx_v_i, __pyx_v_j); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 143, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_4 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 143, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); + __pyx_t_3 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 143, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_XDECREF_SET(__pyx_v_ngram, ((PyObject*)__pyx_t_4)); - __pyx_t_4 = 0; + __Pyx_XDECREF_SET(__pyx_v_ngram, ((PyObject*)__pyx_t_3)); + __pyx_t_3 = 0; /* "gensim/models/_utils_any2vec.pyx":144 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): @@ -2747,10 +2751,10 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P * n += 1 * return ngrams */ - __pyx_t_4 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_v_ngram); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 144, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_4); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 144, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_3 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_v_ngram); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 144, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_13 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_3); if (unlikely(__pyx_t_13 == ((int)-1))) __PYX_ERR(0, 144, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; /* "gensim/models/_utils_any2vec.pyx":142 * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: @@ -2793,8 +2797,8 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = 0; __pyx_L0:; @@ -5382,6 +5386,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, {&__pyx_kp_u__2, __pyx_k__2, sizeof(__pyx_k__2), 0, 1, 0, 0}, {&__pyx_n_s_cline_in_traceback, __pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 0, 1, 1}, + {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, {&__pyx_n_s_max_n, __pyx_k_max_n, sizeof(__pyx_k_max_n), 0, 0, 1, 1}, @@ -5394,9 +5399,11 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_s_numpy_core_multiarray_failed_to, __pyx_k_numpy_core_multiarray_failed_to, sizeof(__pyx_k_numpy_core_multiarray_failed_to), 0, 0, 1, 0}, {&__pyx_kp_s_numpy_core_umath_failed_to_impor, __pyx_k_numpy_core_umath_failed_to_impor, sizeof(__pyx_k_numpy_core_umath_failed_to_impor), 0, 0, 1, 0}, {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, + {&__pyx_kp_s_s, __pyx_k_s, sizeof(__pyx_k_s), 0, 0, 1, 0}, {&__pyx_n_s_six, __pyx_k_six, sizeof(__pyx_k_six), 0, 0, 1, 1}, {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, {&__pyx_kp_u_unknown_dtype_code_in_numpy_pxd, __pyx_k_unknown_dtype_code_in_numpy_pxd, sizeof(__pyx_k_unknown_dtype_code_in_numpy_pxd), 0, 1, 0, 0}, + {&__pyx_kp_s_utf_8, __pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 0, 1, 0}, {&__pyx_n_s_word, __pyx_k_word, sizeof(__pyx_k_word), 0, 0, 1, 1}, {0, 0, 0, 0, 0, 0, 0} }; @@ -6409,128 +6416,6 @@ static int __Pyx_ParseOptionalKeywords( return -1; } -/* GetItemInt */ -static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { - PyObject *r; - if (!j) return NULL; - r = PyObject_GetItem(o, j); - Py_DECREF(j); - return r; -} -static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, - CYTHON_NCP_UNUSED int wraparound, - CYTHON_NCP_UNUSED int boundscheck) { -#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - Py_ssize_t wrapped_i = i; - if (wraparound & unlikely(i < 0)) { - wrapped_i += PyList_GET_SIZE(o); - } - if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyList_GET_SIZE(o)))) { - PyObject *r = PyList_GET_ITEM(o, wrapped_i); - Py_INCREF(r); - return r; - } - return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); -#else - return PySequence_GetItem(o, i); -#endif -} -static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, - CYTHON_NCP_UNUSED int wraparound, - CYTHON_NCP_UNUSED int boundscheck) { -#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - Py_ssize_t wrapped_i = i; - if (wraparound & unlikely(i < 0)) { - wrapped_i += PyTuple_GET_SIZE(o); - } - if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyTuple_GET_SIZE(o)))) { - PyObject *r = PyTuple_GET_ITEM(o, wrapped_i); - Py_INCREF(r); - return r; - } - return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); -#else - return PySequence_GetItem(o, i); -#endif -} -static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, - CYTHON_NCP_UNUSED int wraparound, - CYTHON_NCP_UNUSED int boundscheck) { -#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS && CYTHON_USE_TYPE_SLOTS - if (is_list || PyList_CheckExact(o)) { - Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); - if ((!boundscheck) || (likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o))))) { - PyObject *r = PyList_GET_ITEM(o, n); - Py_INCREF(r); - return r; - } - } - else if (PyTuple_CheckExact(o)) { - Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); - if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyTuple_GET_SIZE(o)))) { - PyObject *r = PyTuple_GET_ITEM(o, n); - Py_INCREF(r); - return r; - } - } else { - PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (likely(m && m->sq_item)) { - if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { - Py_ssize_t l = m->sq_length(o); - if (likely(l >= 0)) { - i += l; - } else { - if (!PyErr_ExceptionMatches(PyExc_OverflowError)) - return NULL; - PyErr_Clear(); - } - } - return m->sq_item(o, i); - } - } -#else - if (is_list || PySequence_Check(o)) { - return PySequence_GetItem(o, i); - } -#endif - return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); -} - -/* GetModuleGlobalName */ -#if CYTHON_USE_DICT_VERSIONS -static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) -#else -static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) -#endif -{ - PyObject *result; -#if !CYTHON_AVOID_BORROWED_REFS -#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 - result = _PyDict_GetItem_KnownHash(__pyx_d, name, ((PyASCIIObject *) name)->hash); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } else if (unlikely(PyErr_Occurred())) { - return NULL; - } -#else - result = PyDict_GetItem(__pyx_d, name); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } -#endif -#else - result = PyObject_GetItem(__pyx_d, name); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } - PyErr_Clear(); -#endif - return __Pyx_GetBuiltinName(name); -} - /* PyCFunctionFastCall */ #if CYTHON_FAST_PYCCALL static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { @@ -6673,6 +6558,35 @@ static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, #endif #endif +/* PyObjectCall2Args */ +static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { + PyObject *args, *result = NULL; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(function)) { + PyObject *args[2] = {arg1, arg2}; + return __Pyx_PyFunction_FastCall(function, args, 2); + } + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(function)) { + PyObject *args[2] = {arg1, arg2}; + return __Pyx_PyCFunction_FastCall(function, args, 2); + } + #endif + args = PyTuple_New(2); + if (unlikely(!args)) goto done; + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 0, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 1, arg2); + Py_INCREF(function); + result = __Pyx_PyObject_Call(function, args, NULL); + Py_DECREF(args); + Py_DECREF(function); +done: + return result; +} + /* PyObjectCallMethO */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { @@ -6733,6 +6647,128 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObjec } #endif +/* GetItemInt */ +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { + PyObject *r; + if (!j) return NULL; + r = PyObject_GetItem(o, j); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyList_GET_SIZE(o); + } + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyList_GET_SIZE(o)))) { + PyObject *r = PyList_GET_ITEM(o, wrapped_i); + Py_INCREF(r); + return r; + } + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +#else + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyTuple_GET_SIZE(o); + } + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyTuple_GET_SIZE(o)))) { + PyObject *r = PyTuple_GET_ITEM(o, wrapped_i); + Py_INCREF(r); + return r; + } + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +#else + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS && CYTHON_USE_TYPE_SLOTS + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); + if ((!boundscheck) || (likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o))))) { + PyObject *r = PyList_GET_ITEM(o, n); + Py_INCREF(r); + return r; + } + } + else if (PyTuple_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); + if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyTuple_GET_SIZE(o)))) { + PyObject *r = PyTuple_GET_ITEM(o, n); + Py_INCREF(r); + return r; + } + } else { + PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; + if (likely(m && m->sq_item)) { + if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { + Py_ssize_t l = m->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + PyErr_Clear(); + } + } + return m->sq_item(o, i); + } + } +#else + if (is_list || PySequence_Check(o)) { + return PySequence_GetItem(o, i); + } +#endif + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +} + +/* GetModuleGlobalName */ +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ + PyObject *result; +#if !CYTHON_AVOID_BORROWED_REFS +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 + result = _PyDict_GetItem_KnownHash(__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } else if (unlikely(PyErr_Occurred())) { + return NULL; + } +#else + result = PyDict_GetItem(__pyx_d, name); + __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } +#endif +#else + result = PyObject_GetItem(__pyx_d, name); + __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); +} + /* PyErrFetchRestore */ #if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { diff --git a/gensim/models/_utils_any2vec.pyx b/gensim/models/_utils_any2vec.pyx index 94622ad567..674cfe8146 100644 --- a/gensim/models/_utils_any2vec.pyx +++ b/gensim/models/_utils_any2vec.pyx @@ -125,7 +125,7 @@ cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): `Original implementation `__ """ - cdef bytes utf8_word = f'<{word}>'.encode("utf-8") + cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") cdef int num_bytes = len(utf8_word) cdef int j, i, n diff --git a/gensim/models/utils_any2vec.py b/gensim/models/utils_any2vec.py index ff697c290a..0e77840137 100644 --- a/gensim/models/utils_any2vec.py +++ b/gensim/models/utils_any2vec.py @@ -161,7 +161,7 @@ def _compute_ngrams_bytes_py(word, min_n, max_n): `Original implementation `__ # noqa: E501 """ - utf8_word = f'<{word}>'.encode("utf-8") + utf8_word = ('<%s>' % word).encode("utf-8") num_bytes = len(utf8_word) n = 0 From d13e4c95dfe61f179286d04cd7f877631f88e055 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Wed, 23 Jan 2019 13:19:22 +1100 Subject: [PATCH 22/32] flake8 --- gensim/test/test_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gensim/test/test_utils.py b/gensim/test/test_utils.py index 82d33470da..3153195092 100644 --- a/gensim/test/test_utils.py +++ b/gensim/test/test_utils.py @@ -400,13 +400,13 @@ def setUp(self): '', '', '' ], - 'тест': ['<те', 'тес', 'ест', 'ст>', '<тес', 'тест', 'ест>', '<тест', 'тест>'], - 'テスト': ['<テス', 'テスト', 'スト>', '<テスト', 'テスト>', '<テスト>'], - '試し': ['<試し', '試し>', '<試し>'], + 'тест': ['<те', 'тес', 'ест', 'ст>', '<тес', 'тест', 'ест>', '<тест', 'тест>'], + 'テスト': ['<テス', 'テスト', 'スト>', '<テスト', 'テスト>', '<テスト>'], + '試し': ['<試し', '試し>', '<試し>'], '🚑🚒🚓🚕': [ '<🚑🚒', '🚑🚒🚓', '🚒🚓🚕', '🚓🚕>', '<🚑🚒🚓', '🚑🚒🚓🚕', '🚒🚓🚕>', '<🚑🚒🚓🚕', '🚑🚒🚓🚕>' - ], + ], } self.expected_bytes = { 'test': [b'', b'est', b'est>', b'st>'], @@ -422,7 +422,7 @@ def setUp(self): 'テスト': [ b'<\xe3\x83\x86\xe3\x82\xb9', b'<\xe3\x83\x86\xe3\x82\xb9\xe3\x83\x88', b'<\xe3\x83\x86\xe3\x82\xb9\xe3\x83\x88>', b'\xe3\x83\x86\xe3\x82\xb9\xe3\x83\x88', - b'\xe3\x83\x86\xe3\x82\xb9\xe3\x83\x88>',b'\xe3\x82\xb9\xe3\x83\x88>' + b'\xe3\x83\x86\xe3\x82\xb9\xe3\x83\x88>', b'\xe3\x82\xb9\xe3\x83\x88>' ], '試し': [b'<\xe8\xa9\xa6\xe3\x81\x97', b'<\xe8\xa9\xa6\xe3\x81\x97>', b'\xe8\xa9\xa6\xe3\x81\x97>'], '🚑🚒🚓🚕': [ From a412e48b03f51e600591134438a2d8cca621e9ea Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Wed, 23 Jan 2019 14:15:20 +1100 Subject: [PATCH 23/32] more Py2 compatibility --- gensim/models/_utils_any2vec.c | 2012 ++++++++++++++++++++++++------ gensim/models/_utils_any2vec.pyx | 15 +- gensim/models/utils_any2vec.py | 13 +- 3 files changed, 1620 insertions(+), 420 deletions(-) diff --git a/gensim/models/_utils_any2vec.c b/gensim/models/_utils_any2vec.c index d4d5caeb09..ad8cdfe979 100644 --- a/gensim/models/_utils_any2vec.c +++ b/gensim/models/_utils_any2vec.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.29.2 */ +/* Generated by Cython 0.29.3 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -7,8 +7,8 @@ #elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03030000) #error Cython requires Python 2.6+ or Python 3.3+. #else -#define CYTHON_ABI "0_29_2" -#define CYTHON_HEX_VERSION 0x001D02F0 +#define CYTHON_ABI "0_29_3" +#define CYTHON_HEX_VERSION 0x001D03F0 #define CYTHON_FUTURE_DIVISION 0 #include #ifndef offsetof @@ -398,7 +398,7 @@ typedef int Py_tss_t; static CYTHON_INLINE int PyThread_tss_create(Py_tss_t *key) { *key = PyThread_create_key(); - return 0; // PyThread_create_key reports success always + return 0; } static CYTHON_INLINE Py_tss_t * PyThread_tss_alloc(void) { Py_tss_t *key = (Py_tss_t *)PyObject_Malloc(sizeof(Py_tss_t)); @@ -421,7 +421,7 @@ static CYTHON_INLINE int PyThread_tss_set(Py_tss_t *key, void *value) { static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { return PyThread_get_key_value(*key); } -#endif // TSS (Thread Specific Storage) API +#endif #if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) #define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) #else @@ -844,10 +844,11 @@ static const char *__pyx_filename; static const char *__pyx_f[] = { "gensim/models/_utils_any2vec.pyx", "__init__.pxd", + "stringsource", "type.pxd", }; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":776 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":776 * # in Cython to enable them only on the right systems. * * ctypedef npy_int8 int8_t # <<<<<<<<<<<<<< @@ -856,7 +857,7 @@ static const char *__pyx_f[] = { */ typedef npy_int8 __pyx_t_5numpy_int8_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":777 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":777 * * ctypedef npy_int8 int8_t * ctypedef npy_int16 int16_t # <<<<<<<<<<<<<< @@ -865,7 +866,7 @@ typedef npy_int8 __pyx_t_5numpy_int8_t; */ typedef npy_int16 __pyx_t_5numpy_int16_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":778 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":778 * ctypedef npy_int8 int8_t * ctypedef npy_int16 int16_t * ctypedef npy_int32 int32_t # <<<<<<<<<<<<<< @@ -874,7 +875,7 @@ typedef npy_int16 __pyx_t_5numpy_int16_t; */ typedef npy_int32 __pyx_t_5numpy_int32_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":779 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":779 * ctypedef npy_int16 int16_t * ctypedef npy_int32 int32_t * ctypedef npy_int64 int64_t # <<<<<<<<<<<<<< @@ -883,7 +884,7 @@ typedef npy_int32 __pyx_t_5numpy_int32_t; */ typedef npy_int64 __pyx_t_5numpy_int64_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":783 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":783 * #ctypedef npy_int128 int128_t * * ctypedef npy_uint8 uint8_t # <<<<<<<<<<<<<< @@ -892,7 +893,7 @@ typedef npy_int64 __pyx_t_5numpy_int64_t; */ typedef npy_uint8 __pyx_t_5numpy_uint8_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":784 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":784 * * ctypedef npy_uint8 uint8_t * ctypedef npy_uint16 uint16_t # <<<<<<<<<<<<<< @@ -901,7 +902,7 @@ typedef npy_uint8 __pyx_t_5numpy_uint8_t; */ typedef npy_uint16 __pyx_t_5numpy_uint16_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":785 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":785 * ctypedef npy_uint8 uint8_t * ctypedef npy_uint16 uint16_t * ctypedef npy_uint32 uint32_t # <<<<<<<<<<<<<< @@ -910,7 +911,7 @@ typedef npy_uint16 __pyx_t_5numpy_uint16_t; */ typedef npy_uint32 __pyx_t_5numpy_uint32_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":786 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":786 * ctypedef npy_uint16 uint16_t * ctypedef npy_uint32 uint32_t * ctypedef npy_uint64 uint64_t # <<<<<<<<<<<<<< @@ -919,7 +920,7 @@ typedef npy_uint32 __pyx_t_5numpy_uint32_t; */ typedef npy_uint64 __pyx_t_5numpy_uint64_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":790 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":790 * #ctypedef npy_uint128 uint128_t * * ctypedef npy_float32 float32_t # <<<<<<<<<<<<<< @@ -928,7 +929,7 @@ typedef npy_uint64 __pyx_t_5numpy_uint64_t; */ typedef npy_float32 __pyx_t_5numpy_float32_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":791 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":791 * * ctypedef npy_float32 float32_t * ctypedef npy_float64 float64_t # <<<<<<<<<<<<<< @@ -937,7 +938,7 @@ typedef npy_float32 __pyx_t_5numpy_float32_t; */ typedef npy_float64 __pyx_t_5numpy_float64_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":800 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":800 * # The int types are mapped a bit surprising -- * # numpy.int corresponds to 'l' and numpy.long to 'q' * ctypedef npy_long int_t # <<<<<<<<<<<<<< @@ -946,7 +947,7 @@ typedef npy_float64 __pyx_t_5numpy_float64_t; */ typedef npy_long __pyx_t_5numpy_int_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":801 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":801 * # numpy.int corresponds to 'l' and numpy.long to 'q' * ctypedef npy_long int_t * ctypedef npy_longlong long_t # <<<<<<<<<<<<<< @@ -955,7 +956,7 @@ typedef npy_long __pyx_t_5numpy_int_t; */ typedef npy_longlong __pyx_t_5numpy_long_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":802 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":802 * ctypedef npy_long int_t * ctypedef npy_longlong long_t * ctypedef npy_longlong longlong_t # <<<<<<<<<<<<<< @@ -964,7 +965,7 @@ typedef npy_longlong __pyx_t_5numpy_long_t; */ typedef npy_longlong __pyx_t_5numpy_longlong_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":804 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":804 * ctypedef npy_longlong longlong_t * * ctypedef npy_ulong uint_t # <<<<<<<<<<<<<< @@ -973,7 +974,7 @@ typedef npy_longlong __pyx_t_5numpy_longlong_t; */ typedef npy_ulong __pyx_t_5numpy_uint_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":805 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":805 * * ctypedef npy_ulong uint_t * ctypedef npy_ulonglong ulong_t # <<<<<<<<<<<<<< @@ -982,7 +983,7 @@ typedef npy_ulong __pyx_t_5numpy_uint_t; */ typedef npy_ulonglong __pyx_t_5numpy_ulong_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":806 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":806 * ctypedef npy_ulong uint_t * ctypedef npy_ulonglong ulong_t * ctypedef npy_ulonglong ulonglong_t # <<<<<<<<<<<<<< @@ -991,7 +992,7 @@ typedef npy_ulonglong __pyx_t_5numpy_ulong_t; */ typedef npy_ulonglong __pyx_t_5numpy_ulonglong_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":808 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":808 * ctypedef npy_ulonglong ulonglong_t * * ctypedef npy_intp intp_t # <<<<<<<<<<<<<< @@ -1000,7 +1001,7 @@ typedef npy_ulonglong __pyx_t_5numpy_ulonglong_t; */ typedef npy_intp __pyx_t_5numpy_intp_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":809 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":809 * * ctypedef npy_intp intp_t * ctypedef npy_uintp uintp_t # <<<<<<<<<<<<<< @@ -1009,7 +1010,7 @@ typedef npy_intp __pyx_t_5numpy_intp_t; */ typedef npy_uintp __pyx_t_5numpy_uintp_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":811 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":811 * ctypedef npy_uintp uintp_t * * ctypedef npy_double float_t # <<<<<<<<<<<<<< @@ -1018,7 +1019,7 @@ typedef npy_uintp __pyx_t_5numpy_uintp_t; */ typedef npy_double __pyx_t_5numpy_float_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":812 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":812 * * ctypedef npy_double float_t * ctypedef npy_double double_t # <<<<<<<<<<<<<< @@ -1027,7 +1028,7 @@ typedef npy_double __pyx_t_5numpy_float_t; */ typedef npy_double __pyx_t_5numpy_double_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":813 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":813 * ctypedef npy_double float_t * ctypedef npy_double double_t * ctypedef npy_longdouble longdouble_t # <<<<<<<<<<<<<< @@ -1061,8 +1062,9 @@ static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(do /*--- Type declarations ---*/ +struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":815 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":815 * ctypedef npy_longdouble longdouble_t * * ctypedef npy_cfloat cfloat_t # <<<<<<<<<<<<<< @@ -1071,7 +1073,7 @@ static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(do */ typedef npy_cfloat __pyx_t_5numpy_cfloat_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":816 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":816 * * ctypedef npy_cfloat cfloat_t * ctypedef npy_cdouble cdouble_t # <<<<<<<<<<<<<< @@ -1080,7 +1082,7 @@ typedef npy_cfloat __pyx_t_5numpy_cfloat_t; */ typedef npy_cdouble __pyx_t_5numpy_cdouble_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":817 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":817 * ctypedef npy_cfloat cfloat_t * ctypedef npy_cdouble cdouble_t * ctypedef npy_clongdouble clongdouble_t # <<<<<<<<<<<<<< @@ -1089,7 +1091,7 @@ typedef npy_cdouble __pyx_t_5numpy_cdouble_t; */ typedef npy_clongdouble __pyx_t_5numpy_clongdouble_t; -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":819 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":819 * ctypedef npy_clongdouble clongdouble_t * * ctypedef npy_cdouble complex_t # <<<<<<<<<<<<<< @@ -1098,6 +1100,19 @@ typedef npy_clongdouble __pyx_t_5numpy_clongdouble_t; */ typedef npy_cdouble __pyx_t_5numpy_complex_t; +/* "cfunc.to_py":64 + * + * @cname("__Pyx_CFunc_object____object___to_py") + * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< + * def wrap(object b): + * """wrap(b)""" + */ +struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py { + PyObject_HEAD + PyObject *(*__pyx_v_f)(PyObject *); +}; + + /* --- Runtime support code (head) --- */ /* Refnanny.proto */ #ifndef CYTHON_REFNANNY @@ -1271,6 +1286,27 @@ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ const char* function_name); +/* GetModuleGlobalName.proto */ +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_d))) ?\ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} +#define __Pyx_GetModuleGlobalNameUncached(var, name) {\ + PY_UINT64_T __pyx_dict_version;\ + PyObject *__pyx_dict_cached_value;\ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); +#else +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); +#endif + /* PyCFunctionFastCall.proto */ #if CYTHON_FAST_PYCCALL static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); @@ -1334,27 +1370,6 @@ static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j); static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, int wraparound, int boundscheck); -/* GetModuleGlobalName.proto */ -#if CYTHON_USE_DICT_VERSIONS -#define __Pyx_GetModuleGlobalName(var, name) {\ - static PY_UINT64_T __pyx_dict_version = 0;\ - static PyObject *__pyx_dict_cached_value = NULL;\ - (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_d))) ?\ - (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ - __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ -} -#define __Pyx_GetModuleGlobalNameUncached(var, name) {\ - PY_UINT64_T __pyx_dict_version;\ - PyObject *__pyx_dict_cached_value;\ - (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ -} -static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); -#else -#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) -#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) -static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); -#endif - /* PyThreadStateGet.proto */ #if CYTHON_FAST_THREAD_STATE #define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; @@ -1449,6 +1464,70 @@ static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); #endif +/* FetchCommonType.proto */ +static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); + +/* CythonFunction.proto */ +#define __Pyx_CyFunction_USED 1 +#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 +#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 +#define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CyFunction_GetClosure(f)\ + (((__pyx_CyFunctionObject *) (f))->func_closure) +#define __Pyx_CyFunction_GetClassObj(f)\ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#define __Pyx_CyFunction_Defaults(type, f)\ + ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) +#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ + ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) +typedef struct { + PyCFunctionObject func; +#if PY_VERSION_HEX < 0x030500A0 + PyObject *func_weakreflist; +#endif + PyObject *func_dict; + PyObject *func_name; + PyObject *func_qualname; + PyObject *func_doc; + PyObject *func_globals; + PyObject *func_code; + PyObject *func_closure; + PyObject *func_classobj; + void *defaults; + int defaults_pyobjects; + int flags; + PyObject *defaults_tuple; + PyObject *defaults_kwdict; + PyObject *(*defaults_getter)(PyObject *); + PyObject *func_annotations; +} __pyx_CyFunctionObject; +static PyTypeObject *__pyx_CyFunctionType = 0; +#define __Pyx_CyFunction_Check(obj) (__Pyx_TypeCheck(obj, __pyx_CyFunctionType)) +#define __Pyx_CyFunction_NewEx(ml, flags, qualname, self, module, globals, code)\ + __Pyx_CyFunction_New(__pyx_CyFunctionType, ml, flags, qualname, self, module, globals, code) +static PyObject *__Pyx_CyFunction_New(PyTypeObject *, PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *self, + PyObject *module, PyObject *globals, + PyObject* code); +static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *m, + size_t size, + int pyobjects); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, + PyObject *tuple); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, + PyObject *dict); +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, + PyObject *dict); +static int __pyx_CyFunction_init(void); + +/* PyObject_GenericGetAttrNoDict.proto */ +#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 +static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GenericGetAttrNoDict PyObject_GenericGetAttr +#endif + /* TypeImport.proto */ #ifndef __PYX_HAVE_RT_ImportType_proto #define __PYX_HAVE_RT_ImportType_proto @@ -1498,6 +1577,9 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint32_t(uint32_t value); /* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value); +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); + /* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); @@ -1611,9 +1693,6 @@ static CYTHON_INLINE char __Pyx_PyInt_As_char(PyObject *); /* CIntFromPy.proto */ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); - /* CIntFromPy.proto */ static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); @@ -1667,10 +1746,14 @@ static PyTypeObject *__pyx_ptype_5numpy_ufunc = 0; static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *, char *, char *, int *); /*proto*/ /* Module declarations from 'gensim.models._utils_any2vec' */ +static PyTypeObject *__pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py = 0; static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject *, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObject *, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObject *, unsigned int, unsigned int, int __pyx_skip_dispatch); /*proto*/ +static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__is_utf8_continue_py3(PyObject *); /*proto*/ +static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__is_utf8_continue_py2(PyObject *); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(PyObject *, unsigned int, unsigned int, int __pyx_skip_dispatch); /*proto*/ +static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*)(PyObject *)); /*proto*/ #define __Pyx_MODULE_NAME "gensim.models._utils_any2vec" extern int __pyx_module_is_main_gensim__models___utils_any2vec; int __pyx_module_is_main_gensim__models___utils_any2vec = 0; @@ -1681,6 +1764,7 @@ static PyObject *__pyx_builtin_ValueError; static PyObject *__pyx_builtin_RuntimeError; static PyObject *__pyx_builtin_ImportError; static const char __pyx_k_[] = "<"; +static const char __pyx_k_b[] = "b"; static const char __pyx_k_s[] = "<%s>"; static const char __pyx_k__2[] = ">"; static const char __pyx_k_np[] = "np"; @@ -1690,6 +1774,7 @@ static const char __pyx_k_main[] = "__main__"; static const char __pyx_k_name[] = "__name__"; static const char __pyx_k_test[] = "__test__"; static const char __pyx_k_word[] = "word"; +static const char __pyx_k_wrap[] = "wrap"; static const char __pyx_k_max_n[] = "max_n"; static const char __pyx_k_min_n[] = "min_n"; static const char __pyx_k_numpy[] = "numpy"; @@ -1701,9 +1786,13 @@ static const char __pyx_k_MB_MASK[] = "_MB_MASK"; static const char __pyx_k_MB_START[] = "_MB_START"; static const char __pyx_k_ValueError[] = "ValueError"; static const char __pyx_k_ImportError[] = "ImportError"; +static const char __pyx_k_cfunc_to_py[] = "cfunc.to_py"; static const char __pyx_k_RuntimeError[] = "RuntimeError"; +static const char __pyx_k_stringsource[] = "stringsource"; +static const char __pyx_k_is_utf8_continue[] = "_is_utf8_continue"; static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; static const char __pyx_k_ndarray_is_not_C_contiguous[] = "ndarray is not C contiguous"; +static const char __pyx_k_Pyx_CFunc_object____object___t[] = "__Pyx_CFunc_object____object___to_py..wrap"; static const char __pyx_k_numpy_core_multiarray_failed_to[] = "numpy.core.multiarray failed to import"; static const char __pyx_k_unknown_dtype_code_in_numpy_pxd[] = "unknown dtype code in numpy.pxd (%d)"; static const char __pyx_k_Format_string_allocated_too_shor[] = "Format string allocated too short, see comment in numpy.pxd"; @@ -1720,12 +1809,16 @@ static PyObject *__pyx_n_s_MB_MASK; static PyObject *__pyx_n_s_MB_START; static PyObject *__pyx_kp_u_Non_native_byte_order_not_suppor; static PyObject *__pyx_n_s_PY2; +static PyObject *__pyx_n_s_Pyx_CFunc_object____object___t; static PyObject *__pyx_n_s_RuntimeError; static PyObject *__pyx_n_s_ValueError; static PyObject *__pyx_kp_u__2; +static PyObject *__pyx_n_s_b; +static PyObject *__pyx_n_s_cfunc_to_py; static PyObject *__pyx_n_s_cline_in_traceback; static PyObject *__pyx_n_s_encode; static PyObject *__pyx_n_s_import; +static PyObject *__pyx_n_s_is_utf8_continue; static PyObject *__pyx_n_s_main; static PyObject *__pyx_n_s_max_n; static PyObject *__pyx_n_s_min_n; @@ -1739,16 +1832,20 @@ static PyObject *__pyx_kp_s_numpy_core_umath_failed_to_impor; static PyObject *__pyx_n_s_range; static PyObject *__pyx_kp_s_s; static PyObject *__pyx_n_s_six; +static PyObject *__pyx_kp_s_stringsource; static PyObject *__pyx_n_s_test; static PyObject *__pyx_kp_u_unknown_dtype_code_in_numpy_pxd; static PyObject *__pyx_kp_s_utf_8; static PyObject *__pyx_n_s_word; +static PyObject *__pyx_n_s_wrap; static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_bytez); /* proto */ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string); /* proto */ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info); /* proto */ +static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(PyObject *__pyx_self, PyObject *__pyx_v_b); /* proto */ +static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_int_0; static PyObject *__pyx_int_1; static PyObject *__pyx_int_128; @@ -1760,6 +1857,8 @@ static PyObject *__pyx_tuple__6; static PyObject *__pyx_tuple__7; static PyObject *__pyx_tuple__8; static PyObject *__pyx_tuple__9; +static PyObject *__pyx_tuple__10; +static PyObject *__pyx_codeobj__11; /* Late includes */ /* "gensim/models/_utils_any2vec.pyx":22 @@ -2465,6 +2564,129 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHO } /* "gensim/models/_utils_any2vec.pyx":104 + * + * + * cdef _is_utf8_continue_py3(b): # <<<<<<<<<<<<<< + * return b & _MB_MASK == _MB_START + * + */ + +static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__is_utf8_continue_py3(PyObject *__pyx_v_b) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + __Pyx_RefNannySetupContext("_is_utf8_continue_py3", 0); + + /* "gensim/models/_utils_any2vec.pyx":105 + * + * cdef _is_utf8_continue_py3(b): + * return b & _MB_MASK == _MB_START # <<<<<<<<<<<<<< + * + * + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 105, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyNumber_And(__pyx_v_b, __pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 105, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 105, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = PyObject_RichCompare(__pyx_t_2, __pyx_t_1, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 105, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "gensim/models/_utils_any2vec.pyx":104 + * + * + * cdef _is_utf8_continue_py3(b): # <<<<<<<<<<<<<< + * return b & _MB_MASK == _MB_START + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("gensim.models._utils_any2vec._is_utf8_continue_py3", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "gensim/models/_utils_any2vec.pyx":108 + * + * + * cdef _is_utf8_continue_py2(b): # <<<<<<<<<<<<<< + * return ord(b) & _MB_MASK == _MB_START + * + */ + +static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__is_utf8_continue_py2(PyObject *__pyx_v_b) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + long __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + __Pyx_RefNannySetupContext("_is_utf8_continue_py2", 0); + + /* "gensim/models/_utils_any2vec.pyx":109 + * + * cdef _is_utf8_continue_py2(b): + * return ord(b) & _MB_MASK == _MB_START # <<<<<<<<<<<<<< + * + * + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_Ord(__pyx_v_b); if (unlikely(__pyx_t_1 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 109, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 109, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 109, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyNumber_And(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 109, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 109, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = PyObject_RichCompare(__pyx_t_4, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 109, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "gensim/models/_utils_any2vec.pyx":108 + * + * + * cdef _is_utf8_continue_py2(b): # <<<<<<<<<<<<<< + * return ord(b) & _MB_MASK == _MB_START + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("gensim.models._utils_any2vec._is_utf8_continue_py2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "gensim/models/_utils_any2vec.pyx":115 * * * cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -2490,24 +2712,25 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P int __pyx_t_5; int __pyx_t_6; int __pyx_t_7; - int __pyx_t_8; + PyObject *__pyx_t_8 = NULL; int __pyx_t_9; int __pyx_t_10; int __pyx_t_11; int __pyx_t_12; int __pyx_t_13; + int __pyx_t_14; __Pyx_RefNannySetupContext("compute_ngrams_bytes", 0); - /* "gensim/models/_utils_any2vec.pyx":128 + /* "gensim/models/_utils_any2vec.pyx":139 * * """ * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") # <<<<<<<<<<<<<< * cdef int num_bytes = len(utf8_word) * cdef int j, i, n */ - __pyx_t_2 = __Pyx_PyString_FormatSafe(__pyx_kp_s_s, __pyx_v_word); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 128, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyString_FormatSafe(__pyx_kp_s_s, __pyx_v_word); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 139, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 128, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 139, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = NULL; @@ -2522,14 +2745,14 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P } __pyx_t_1 = (__pyx_t_2) ? __Pyx_PyObject_Call2Args(__pyx_t_3, __pyx_t_2, __pyx_kp_s_utf_8) : __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_kp_s_utf_8); __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; - if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 128, __pyx_L1_error) + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 139, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 128, __pyx_L1_error) + if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 139, __pyx_L1_error) __pyx_v_utf8_word = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":129 + /* "gensim/models/_utils_any2vec.pyx":140 * """ * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") * cdef int num_bytes = len(utf8_word) # <<<<<<<<<<<<<< @@ -2538,28 +2761,28 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P */ if (unlikely(__pyx_v_utf8_word == Py_None)) { PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); - __PYX_ERR(0, 129, __pyx_L1_error) + __PYX_ERR(0, 140, __pyx_L1_error) } - __pyx_t_4 = PyBytes_GET_SIZE(__pyx_v_utf8_word); if (unlikely(__pyx_t_4 == ((Py_ssize_t)-1))) __PYX_ERR(0, 129, __pyx_L1_error) + __pyx_t_4 = PyBytes_GET_SIZE(__pyx_v_utf8_word); if (unlikely(__pyx_t_4 == ((Py_ssize_t)-1))) __PYX_ERR(0, 140, __pyx_L1_error) __pyx_v_num_bytes = __pyx_t_4; - /* "gensim/models/_utils_any2vec.pyx":132 + /* "gensim/models/_utils_any2vec.pyx":143 * cdef int j, i, n * * ngrams = [] # <<<<<<<<<<<<<< * for i in range(num_bytes): - * if utf8_word[i] & _MB_MASK == _MB_START: + * if _is_utf8_continue(utf8_word[i]): */ - __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 132, __pyx_L1_error) + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 143, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_ngrams = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":133 + /* "gensim/models/_utils_any2vec.pyx":144 * * ngrams = [] * for i in range(num_bytes): # <<<<<<<<<<<<<< - * if utf8_word[i] & _MB_MASK == _MB_START: + * if _is_utf8_continue(utf8_word[i]): * continue */ __pyx_t_5 = __pyx_v_num_bytes; @@ -2567,124 +2790,138 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_6; __pyx_t_7+=1) { __pyx_v_i = __pyx_t_7; - /* "gensim/models/_utils_any2vec.pyx":134 + /* "gensim/models/_utils_any2vec.pyx":145 * ngrams = [] * for i in range(num_bytes): - * if utf8_word[i] & _MB_MASK == _MB_START: # <<<<<<<<<<<<<< + * if _is_utf8_continue(utf8_word[i]): # <<<<<<<<<<<<<< * continue * */ - __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 134, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 134, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_is_utf8_continue); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_2 = PyNumber_And(__pyx_t_1, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 134, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 134, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_RichCompare(__pyx_t_2, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 134, __pyx_L1_error) + __pyx_t_8 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_8)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_8); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + } + } + __pyx_t_1 = (__pyx_t_8) ? __Pyx_PyObject_Call2Args(__pyx_t_3, __pyx_t_8, __pyx_t_2) : __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 145, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_8 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_8 < 0)) __PYX_ERR(0, 134, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_9 < 0)) __PYX_ERR(0, 145, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (__pyx_t_8) { + if (__pyx_t_9) { - /* "gensim/models/_utils_any2vec.pyx":135 + /* "gensim/models/_utils_any2vec.pyx":146 * for i in range(num_bytes): - * if utf8_word[i] & _MB_MASK == _MB_START: + * if _is_utf8_continue(utf8_word[i]): * continue # <<<<<<<<<<<<<< * * j, n = i, 1 */ goto __pyx_L3_continue; - /* "gensim/models/_utils_any2vec.pyx":134 + /* "gensim/models/_utils_any2vec.pyx":145 * ngrams = [] * for i in range(num_bytes): - * if utf8_word[i] & _MB_MASK == _MB_START: # <<<<<<<<<<<<<< + * if _is_utf8_continue(utf8_word[i]): # <<<<<<<<<<<<<< * continue * */ } - /* "gensim/models/_utils_any2vec.pyx":137 + /* "gensim/models/_utils_any2vec.pyx":148 * continue * * j, n = i, 1 # <<<<<<<<<<<<<< * while j < num_bytes and n <= max_n: * j += 1 */ - __pyx_t_9 = __pyx_v_i; - __pyx_t_10 = 1; - __pyx_v_j = __pyx_t_9; - __pyx_v_n = __pyx_t_10; + __pyx_t_10 = __pyx_v_i; + __pyx_t_11 = 1; + __pyx_v_j = __pyx_t_10; + __pyx_v_n = __pyx_t_11; - /* "gensim/models/_utils_any2vec.pyx":138 + /* "gensim/models/_utils_any2vec.pyx":149 * * j, n = i, 1 * while j < num_bytes and n <= max_n: # <<<<<<<<<<<<<< * j += 1 - * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: + * while j < num_bytes and _is_utf8_continue(utf8_word[j]): */ while (1) { - __pyx_t_11 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); - if (__pyx_t_11) { + __pyx_t_12 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); + if (__pyx_t_12) { } else { - __pyx_t_8 = __pyx_t_11; + __pyx_t_9 = __pyx_t_12; goto __pyx_L8_bool_binop_done; } - __pyx_t_11 = ((__pyx_v_n <= __pyx_v_max_n) != 0); - __pyx_t_8 = __pyx_t_11; + __pyx_t_12 = ((__pyx_v_n <= __pyx_v_max_n) != 0); + __pyx_t_9 = __pyx_t_12; __pyx_L8_bool_binop_done:; - if (!__pyx_t_8) break; + if (!__pyx_t_9) break; - /* "gensim/models/_utils_any2vec.pyx":139 + /* "gensim/models/_utils_any2vec.pyx":150 * j, n = i, 1 * while j < num_bytes and n <= max_n: * j += 1 # <<<<<<<<<<<<<< - * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: + * while j < num_bytes and _is_utf8_continue(utf8_word[j]): * j += 1 */ __pyx_v_j = (__pyx_v_j + 1); - /* "gensim/models/_utils_any2vec.pyx":140 + /* "gensim/models/_utils_any2vec.pyx":151 * while j < num_bytes and n <= max_n: * j += 1 - * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: # <<<<<<<<<<<<<< + * while j < num_bytes and _is_utf8_continue(utf8_word[j]): # <<<<<<<<<<<<<< * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): */ while (1) { - __pyx_t_11 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); - if (__pyx_t_11) { + __pyx_t_12 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); + if (__pyx_t_12) { } else { - __pyx_t_8 = __pyx_t_11; + __pyx_t_9 = __pyx_t_12; goto __pyx_L12_bool_binop_done; } - __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 140, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 140, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_is_utf8_continue); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 151, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_2 = PyNumber_And(__pyx_t_1, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 140, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 151, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 140, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_RichCompare(__pyx_t_2, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 140, __pyx_L1_error) + __pyx_t_8 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_8)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_8); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + } + } + __pyx_t_1 = (__pyx_t_8) ? __Pyx_PyObject_Call2Args(__pyx_t_3, __pyx_t_8, __pyx_t_2) : __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 151, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_11 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_11 < 0)) __PYX_ERR(0, 140, __pyx_L1_error) + __pyx_t_12 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_12 < 0)) __PYX_ERR(0, 151, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_8 = __pyx_t_11; + __pyx_t_9 = __pyx_t_12; __pyx_L12_bool_binop_done:; - if (!__pyx_t_8) break; + if (!__pyx_t_9) break; - /* "gensim/models/_utils_any2vec.pyx":141 + /* "gensim/models/_utils_any2vec.pyx":152 * j += 1 - * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: + * while j < num_bytes and _is_utf8_continue(utf8_word[j]): * j += 1 # <<<<<<<<<<<<<< * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): * ngram = bytes(utf8_word[i:j]) @@ -2692,40 +2929,40 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_v_j = (__pyx_v_j + 1); } - /* "gensim/models/_utils_any2vec.pyx":142 - * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: + /* "gensim/models/_utils_any2vec.pyx":153 + * while j < num_bytes and _is_utf8_continue(utf8_word[j]): * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< * ngram = bytes(utf8_word[i:j]) * ngrams.append(bytes(ngram)) */ - __pyx_t_11 = ((__pyx_v_n >= __pyx_v_min_n) != 0); - if (__pyx_t_11) { + __pyx_t_12 = ((__pyx_v_n >= __pyx_v_min_n) != 0); + if (__pyx_t_12) { } else { - __pyx_t_8 = __pyx_t_11; + __pyx_t_9 = __pyx_t_12; goto __pyx_L15_bool_binop_done; } - __pyx_t_12 = ((__pyx_v_n == 1) != 0); - if (__pyx_t_12) { + __pyx_t_13 = ((__pyx_v_n == 1) != 0); + if (__pyx_t_13) { } else { - __pyx_t_11 = __pyx_t_12; + __pyx_t_12 = __pyx_t_13; goto __pyx_L17_bool_binop_done; } - __pyx_t_12 = ((__pyx_v_i == 0) != 0); - if (!__pyx_t_12) { + __pyx_t_13 = ((__pyx_v_i == 0) != 0); + if (!__pyx_t_13) { } else { - __pyx_t_11 = __pyx_t_12; + __pyx_t_12 = __pyx_t_13; goto __pyx_L17_bool_binop_done; } - __pyx_t_12 = ((__pyx_v_j == __pyx_v_num_bytes) != 0); - __pyx_t_11 = __pyx_t_12; + __pyx_t_13 = ((__pyx_v_j == __pyx_v_num_bytes) != 0); + __pyx_t_12 = __pyx_t_13; __pyx_L17_bool_binop_done:; - __pyx_t_12 = ((!__pyx_t_11) != 0); - __pyx_t_8 = __pyx_t_12; + __pyx_t_13 = ((!__pyx_t_12) != 0); + __pyx_t_9 = __pyx_t_13; __pyx_L15_bool_binop_done:; - if (__pyx_t_8) { + if (__pyx_t_9) { - /* "gensim/models/_utils_any2vec.pyx":143 + /* "gensim/models/_utils_any2vec.pyx":154 * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): * ngram = bytes(utf8_word[i:j]) # <<<<<<<<<<<<<< @@ -2734,30 +2971,30 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P */ if (unlikely(__pyx_v_utf8_word == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); - __PYX_ERR(0, 143, __pyx_L1_error) + __PYX_ERR(0, 154, __pyx_L1_error) } - __pyx_t_1 = PySequence_GetSlice(__pyx_v_utf8_word, __pyx_v_i, __pyx_v_j); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 143, __pyx_L1_error) + __pyx_t_1 = PySequence_GetSlice(__pyx_v_utf8_word, __pyx_v_i, __pyx_v_j); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 154, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 143, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 154, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_XDECREF_SET(__pyx_v_ngram, ((PyObject*)__pyx_t_3)); __pyx_t_3 = 0; - /* "gensim/models/_utils_any2vec.pyx":144 + /* "gensim/models/_utils_any2vec.pyx":155 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): * ngram = bytes(utf8_word[i:j]) * ngrams.append(bytes(ngram)) # <<<<<<<<<<<<<< * n += 1 * return ngrams */ - __pyx_t_3 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_v_ngram); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 144, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_v_ngram); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 155, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_13 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_3); if (unlikely(__pyx_t_13 == ((int)-1))) __PYX_ERR(0, 144, __pyx_L1_error) + __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_3); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 155, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "gensim/models/_utils_any2vec.pyx":142 - * while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: + /* "gensim/models/_utils_any2vec.pyx":153 + * while j < num_bytes and _is_utf8_continue(utf8_word[j]): * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< * ngram = bytes(utf8_word[i:j]) @@ -2765,7 +3002,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P */ } - /* "gensim/models/_utils_any2vec.pyx":145 + /* "gensim/models/_utils_any2vec.pyx":156 * ngram = bytes(utf8_word[i:j]) * ngrams.append(bytes(ngram)) * n += 1 # <<<<<<<<<<<<<< @@ -2776,7 +3013,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_L3_continue:; } - /* "gensim/models/_utils_any2vec.pyx":146 + /* "gensim/models/_utils_any2vec.pyx":157 * ngrams.append(bytes(ngram)) * n += 1 * return ngrams # <<<<<<<<<<<<<< @@ -2786,7 +3023,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_r = __pyx_v_ngrams; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":104 + /* "gensim/models/_utils_any2vec.pyx":115 * * * cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -2799,6 +3036,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __Pyx_XDECREF(__pyx_t_1); __Pyx_XDECREF(__pyx_t_2); __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_8); __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = 0; __pyx_L0:; @@ -2845,17 +3083,17 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes case 1: if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_min_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 1); __PYX_ERR(0, 104, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 1); __PYX_ERR(0, 115, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 2); __PYX_ERR(0, 104, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 2); __PYX_ERR(0, 115, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams_bytes") < 0)) __PYX_ERR(0, 104, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams_bytes") < 0)) __PYX_ERR(0, 115, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { goto __pyx_L5_argtuple_error; @@ -2865,12 +3103,12 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes values[2] = PyTuple_GET_ITEM(__pyx_args, 2); } __pyx_v_word = values[0]; - __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 104, __pyx_L3_error) - __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 104, __pyx_L3_error) + __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 115, __pyx_L3_error) + __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 115, __pyx_L3_error) } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 104, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 115, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -2889,7 +3127,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("compute_ngrams_bytes", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 104, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 115, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -2906,7 +3144,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes return __pyx_r; } -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 * # experimental exception made for __getbuffer__ and __releasebuffer__ * # -- the details of this may change. * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< @@ -2955,7 +3193,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); __Pyx_GIVEREF(__pyx_v_info->obj); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":265 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":265 * * cdef int i, ndim * cdef int endian_detector = 1 # <<<<<<<<<<<<<< @@ -2964,7 +3202,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_endian_detector = 1; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":266 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":266 * cdef int i, ndim * cdef int endian_detector = 1 * cdef bint little_endian = ((&endian_detector)[0] != 0) # <<<<<<<<<<<<<< @@ -2973,7 +3211,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_little_endian = ((((char *)(&__pyx_v_endian_detector))[0]) != 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":268 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":268 * cdef bint little_endian = ((&endian_detector)[0] != 0) * * ndim = PyArray_NDIM(self) # <<<<<<<<<<<<<< @@ -2982,7 +3220,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_ndim = PyArray_NDIM(__pyx_v_self); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 * ndim = PyArray_NDIM(self) * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -2996,7 +3234,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P goto __pyx_L4_bool_binop_done; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":271 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":271 * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): # <<<<<<<<<<<<<< @@ -3007,7 +3245,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = __pyx_t_2; __pyx_L4_bool_binop_done:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 * ndim = PyArray_NDIM(self) * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -3016,7 +3254,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ if (unlikely(__pyx_t_1)) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): * raise ValueError(u"ndarray is not C contiguous") # <<<<<<<<<<<<<< @@ -3029,7 +3267,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 272, __pyx_L1_error) - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 * ndim = PyArray_NDIM(self) * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -3038,7 +3276,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 * raise ValueError(u"ndarray is not C contiguous") * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -3052,7 +3290,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P goto __pyx_L7_bool_binop_done; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":275 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":275 * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): # <<<<<<<<<<<<<< @@ -3063,7 +3301,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = __pyx_t_2; __pyx_L7_bool_binop_done:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 * raise ValueError(u"ndarray is not C contiguous") * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -3072,7 +3310,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ if (unlikely(__pyx_t_1)) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): * raise ValueError(u"ndarray is not Fortran contiguous") # <<<<<<<<<<<<<< @@ -3085,7 +3323,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 276, __pyx_L1_error) - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 * raise ValueError(u"ndarray is not C contiguous") * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -3094,7 +3332,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":278 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":278 * raise ValueError(u"ndarray is not Fortran contiguous") * * info.buf = PyArray_DATA(self) # <<<<<<<<<<<<<< @@ -3103,7 +3341,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->buf = PyArray_DATA(__pyx_v_self); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":279 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":279 * * info.buf = PyArray_DATA(self) * info.ndim = ndim # <<<<<<<<<<<<<< @@ -3112,7 +3350,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->ndim = __pyx_v_ndim; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 * info.buf = PyArray_DATA(self) * info.ndim = ndim * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< @@ -3122,7 +3360,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = (((sizeof(npy_intp)) != (sizeof(Py_ssize_t))) != 0); if (__pyx_t_1) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":283 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":283 * # Allocate new buffer for strides and shape info. * # This is allocated as one block, strides first. * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) # <<<<<<<<<<<<<< @@ -3131,7 +3369,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->strides = ((Py_ssize_t *)PyObject_Malloc((((sizeof(Py_ssize_t)) * 2) * ((size_t)__pyx_v_ndim)))); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":284 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":284 * # This is allocated as one block, strides first. * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) * info.shape = info.strides + ndim # <<<<<<<<<<<<<< @@ -3140,7 +3378,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->shape = (__pyx_v_info->strides + __pyx_v_ndim); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":285 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":285 * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) * info.shape = info.strides + ndim * for i in range(ndim): # <<<<<<<<<<<<<< @@ -3152,7 +3390,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { __pyx_v_i = __pyx_t_6; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":286 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":286 * info.shape = info.strides + ndim * for i in range(ndim): * info.strides[i] = PyArray_STRIDES(self)[i] # <<<<<<<<<<<<<< @@ -3161,7 +3399,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ (__pyx_v_info->strides[__pyx_v_i]) = (PyArray_STRIDES(__pyx_v_self)[__pyx_v_i]); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":287 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":287 * for i in range(ndim): * info.strides[i] = PyArray_STRIDES(self)[i] * info.shape[i] = PyArray_DIMS(self)[i] # <<<<<<<<<<<<<< @@ -3171,7 +3409,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P (__pyx_v_info->shape[__pyx_v_i]) = (PyArray_DIMS(__pyx_v_self)[__pyx_v_i]); } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 * info.buf = PyArray_DATA(self) * info.ndim = ndim * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< @@ -3181,7 +3419,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P goto __pyx_L9; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":289 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":289 * info.shape[i] = PyArray_DIMS(self)[i] * else: * info.strides = PyArray_STRIDES(self) # <<<<<<<<<<<<<< @@ -3191,7 +3429,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P /*else*/ { __pyx_v_info->strides = ((Py_ssize_t *)PyArray_STRIDES(__pyx_v_self)); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":290 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":290 * else: * info.strides = PyArray_STRIDES(self) * info.shape = PyArray_DIMS(self) # <<<<<<<<<<<<<< @@ -3202,7 +3440,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P } __pyx_L9:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":291 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":291 * info.strides = PyArray_STRIDES(self) * info.shape = PyArray_DIMS(self) * info.suboffsets = NULL # <<<<<<<<<<<<<< @@ -3211,7 +3449,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->suboffsets = NULL; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":292 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":292 * info.shape = PyArray_DIMS(self) * info.suboffsets = NULL * info.itemsize = PyArray_ITEMSIZE(self) # <<<<<<<<<<<<<< @@ -3220,7 +3458,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->itemsize = PyArray_ITEMSIZE(__pyx_v_self); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":293 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":293 * info.suboffsets = NULL * info.itemsize = PyArray_ITEMSIZE(self) * info.readonly = not PyArray_ISWRITEABLE(self) # <<<<<<<<<<<<<< @@ -3229,7 +3467,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->readonly = (!(PyArray_ISWRITEABLE(__pyx_v_self) != 0)); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":296 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":296 * * cdef int t * cdef char* f = NULL # <<<<<<<<<<<<<< @@ -3238,7 +3476,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_f = NULL; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":297 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":297 * cdef int t * cdef char* f = NULL * cdef dtype descr = PyArray_DESCR(self) # <<<<<<<<<<<<<< @@ -3251,7 +3489,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_v_descr = ((PyArray_Descr *)__pyx_t_3); __pyx_t_3 = 0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":300 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":300 * cdef int offset * * info.obj = self # <<<<<<<<<<<<<< @@ -3264,7 +3502,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = ((PyObject *)__pyx_v_self); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 * info.obj = self * * if not PyDataType_HASFIELDS(descr): # <<<<<<<<<<<<<< @@ -3274,7 +3512,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = ((!(PyDataType_HASFIELDS(__pyx_v_descr) != 0)) != 0); if (__pyx_t_1) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":303 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":303 * * if not PyDataType_HASFIELDS(descr): * t = descr.type_num # <<<<<<<<<<<<<< @@ -3284,7 +3522,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_4 = __pyx_v_descr->type_num; __pyx_v_t = __pyx_t_4; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 * if not PyDataType_HASFIELDS(descr): * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -3304,7 +3542,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P } __pyx_L15_next_or:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":305 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":305 * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or * (descr.byteorder == c'<' and not little_endian)): # <<<<<<<<<<<<<< @@ -3321,7 +3559,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = __pyx_t_2; __pyx_L14_bool_binop_done:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 * if not PyDataType_HASFIELDS(descr): * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -3330,7 +3568,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ if (unlikely(__pyx_t_1)) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 * if ((descr.byteorder == c'>' and little_endian) or * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< @@ -3343,7 +3581,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 306, __pyx_L1_error) - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 * if not PyDataType_HASFIELDS(descr): * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -3352,7 +3590,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":307 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":307 * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") * if t == NPY_BYTE: f = "b" # <<<<<<<<<<<<<< @@ -3365,7 +3603,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_UBYTE: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":308 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":308 * raise ValueError(u"Non-native byte order not supported") * if t == NPY_BYTE: f = "b" * elif t == NPY_UBYTE: f = "B" # <<<<<<<<<<<<<< @@ -3376,7 +3614,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_SHORT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":309 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":309 * if t == NPY_BYTE: f = "b" * elif t == NPY_UBYTE: f = "B" * elif t == NPY_SHORT: f = "h" # <<<<<<<<<<<<<< @@ -3387,7 +3625,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_USHORT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":310 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":310 * elif t == NPY_UBYTE: f = "B" * elif t == NPY_SHORT: f = "h" * elif t == NPY_USHORT: f = "H" # <<<<<<<<<<<<<< @@ -3398,7 +3636,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_INT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":311 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":311 * elif t == NPY_SHORT: f = "h" * elif t == NPY_USHORT: f = "H" * elif t == NPY_INT: f = "i" # <<<<<<<<<<<<<< @@ -3409,7 +3647,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_UINT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":312 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":312 * elif t == NPY_USHORT: f = "H" * elif t == NPY_INT: f = "i" * elif t == NPY_UINT: f = "I" # <<<<<<<<<<<<<< @@ -3420,7 +3658,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_LONG: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":313 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":313 * elif t == NPY_INT: f = "i" * elif t == NPY_UINT: f = "I" * elif t == NPY_LONG: f = "l" # <<<<<<<<<<<<<< @@ -3431,7 +3669,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_ULONG: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":314 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":314 * elif t == NPY_UINT: f = "I" * elif t == NPY_LONG: f = "l" * elif t == NPY_ULONG: f = "L" # <<<<<<<<<<<<<< @@ -3442,7 +3680,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_LONGLONG: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":315 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":315 * elif t == NPY_LONG: f = "l" * elif t == NPY_ULONG: f = "L" * elif t == NPY_LONGLONG: f = "q" # <<<<<<<<<<<<<< @@ -3453,7 +3691,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_ULONGLONG: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":316 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":316 * elif t == NPY_ULONG: f = "L" * elif t == NPY_LONGLONG: f = "q" * elif t == NPY_ULONGLONG: f = "Q" # <<<<<<<<<<<<<< @@ -3464,7 +3702,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_FLOAT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":317 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":317 * elif t == NPY_LONGLONG: f = "q" * elif t == NPY_ULONGLONG: f = "Q" * elif t == NPY_FLOAT: f = "f" # <<<<<<<<<<<<<< @@ -3475,7 +3713,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_DOUBLE: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":318 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":318 * elif t == NPY_ULONGLONG: f = "Q" * elif t == NPY_FLOAT: f = "f" * elif t == NPY_DOUBLE: f = "d" # <<<<<<<<<<<<<< @@ -3486,7 +3724,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_LONGDOUBLE: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":319 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":319 * elif t == NPY_FLOAT: f = "f" * elif t == NPY_DOUBLE: f = "d" * elif t == NPY_LONGDOUBLE: f = "g" # <<<<<<<<<<<<<< @@ -3497,7 +3735,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_CFLOAT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":320 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":320 * elif t == NPY_DOUBLE: f = "d" * elif t == NPY_LONGDOUBLE: f = "g" * elif t == NPY_CFLOAT: f = "Zf" # <<<<<<<<<<<<<< @@ -3508,7 +3746,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_CDOUBLE: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":321 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":321 * elif t == NPY_LONGDOUBLE: f = "g" * elif t == NPY_CFLOAT: f = "Zf" * elif t == NPY_CDOUBLE: f = "Zd" # <<<<<<<<<<<<<< @@ -3519,7 +3757,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_CLONGDOUBLE: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":322 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":322 * elif t == NPY_CFLOAT: f = "Zf" * elif t == NPY_CDOUBLE: f = "Zd" * elif t == NPY_CLONGDOUBLE: f = "Zg" # <<<<<<<<<<<<<< @@ -3530,7 +3768,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_OBJECT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":323 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":323 * elif t == NPY_CDOUBLE: f = "Zd" * elif t == NPY_CLONGDOUBLE: f = "Zg" * elif t == NPY_OBJECT: f = "O" # <<<<<<<<<<<<<< @@ -3541,7 +3779,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; default: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":325 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":325 * elif t == NPY_OBJECT: f = "O" * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) # <<<<<<<<<<<<<< @@ -3562,7 +3800,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":326 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":326 * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) * info.format = f # <<<<<<<<<<<<<< @@ -3571,7 +3809,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->format = __pyx_v_f; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":327 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":327 * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) * info.format = f * return # <<<<<<<<<<<<<< @@ -3581,7 +3819,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_r = 0; goto __pyx_L0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 * info.obj = self * * if not PyDataType_HASFIELDS(descr): # <<<<<<<<<<<<<< @@ -3590,7 +3828,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":329 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":329 * return * else: * info.format = PyObject_Malloc(_buffer_format_string_len) # <<<<<<<<<<<<<< @@ -3600,7 +3838,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P /*else*/ { __pyx_v_info->format = ((char *)PyObject_Malloc(0xFF)); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":330 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":330 * else: * info.format = PyObject_Malloc(_buffer_format_string_len) * info.format[0] = c'^' # Native data types, manual alignment # <<<<<<<<<<<<<< @@ -3609,7 +3847,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ (__pyx_v_info->format[0]) = '^'; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":331 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":331 * info.format = PyObject_Malloc(_buffer_format_string_len) * info.format[0] = c'^' # Native data types, manual alignment * offset = 0 # <<<<<<<<<<<<<< @@ -3618,7 +3856,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_offset = 0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":332 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":332 * info.format[0] = c'^' # Native data types, manual alignment * offset = 0 * f = _util_dtypestring(descr, info.format + 1, # <<<<<<<<<<<<<< @@ -3628,7 +3866,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_9 = __pyx_f_5numpy__util_dtypestring(__pyx_v_descr, (__pyx_v_info->format + 1), (__pyx_v_info->format + 0xFF), (&__pyx_v_offset)); if (unlikely(__pyx_t_9 == ((char *)NULL))) __PYX_ERR(1, 332, __pyx_L1_error) __pyx_v_f = __pyx_t_9; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":335 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":335 * info.format + _buffer_format_string_len, * &offset) * f[0] = c'\0' # Terminate format string # <<<<<<<<<<<<<< @@ -3638,7 +3876,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P (__pyx_v_f[0]) = '\x00'; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 * # experimental exception made for __getbuffer__ and __releasebuffer__ * # -- the details of this may change. * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< @@ -3670,7 +3908,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P return __pyx_r; } -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 * f[0] = c'\0' # Terminate format string * * def __releasebuffer__(ndarray self, Py_buffer* info): # <<<<<<<<<<<<<< @@ -3694,7 +3932,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s int __pyx_t_1; __Pyx_RefNannySetupContext("__releasebuffer__", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 * * def __releasebuffer__(ndarray self, Py_buffer* info): * if PyArray_HASFIELDS(self): # <<<<<<<<<<<<<< @@ -3704,7 +3942,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s __pyx_t_1 = (PyArray_HASFIELDS(__pyx_v_self) != 0); if (__pyx_t_1) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":339 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":339 * def __releasebuffer__(ndarray self, Py_buffer* info): * if PyArray_HASFIELDS(self): * PyObject_Free(info.format) # <<<<<<<<<<<<<< @@ -3713,7 +3951,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s */ PyObject_Free(__pyx_v_info->format); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 * * def __releasebuffer__(ndarray self, Py_buffer* info): * if PyArray_HASFIELDS(self): # <<<<<<<<<<<<<< @@ -3722,7 +3960,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s */ } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 * if PyArray_HASFIELDS(self): * PyObject_Free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< @@ -3732,7 +3970,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s __pyx_t_1 = (((sizeof(npy_intp)) != (sizeof(Py_ssize_t))) != 0); if (__pyx_t_1) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":341 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":341 * PyObject_Free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): * PyObject_Free(info.strides) # <<<<<<<<<<<<<< @@ -3741,7 +3979,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s */ PyObject_Free(__pyx_v_info->strides); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 * if PyArray_HASFIELDS(self): * PyObject_Free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< @@ -3750,7 +3988,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s */ } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 * f[0] = c'\0' # Terminate format string * * def __releasebuffer__(ndarray self, Py_buffer* info): # <<<<<<<<<<<<<< @@ -3762,7 +4000,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s __Pyx_RefNannyFinishContext(); } -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 * ctypedef npy_cdouble complex_t * * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< @@ -3776,7 +4014,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew1", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":822 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":822 * * cdef inline object PyArray_MultiIterNew1(a): * return PyArray_MultiIterNew(1, a) # <<<<<<<<<<<<<< @@ -3790,7 +4028,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 * ctypedef npy_cdouble complex_t * * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< @@ -3809,7 +4047,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__ return __pyx_r; } -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 * return PyArray_MultiIterNew(1, a) * * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< @@ -3823,7 +4061,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew2", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":825 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":825 * * cdef inline object PyArray_MultiIterNew2(a, b): * return PyArray_MultiIterNew(2, a, b) # <<<<<<<<<<<<<< @@ -3837,7 +4075,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 * return PyArray_MultiIterNew(1, a) * * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< @@ -3856,7 +4094,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__ return __pyx_r; } -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":827 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":827 * return PyArray_MultiIterNew(2, a, b) * * cdef inline object PyArray_MultiIterNew3(a, b, c): # <<<<<<<<<<<<<< @@ -3870,7 +4108,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew3(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew3", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":828 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":828 * * cdef inline object PyArray_MultiIterNew3(a, b, c): * return PyArray_MultiIterNew(3, a, b, c) # <<<<<<<<<<<<<< @@ -3884,7 +4122,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew3(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":827 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":827 * return PyArray_MultiIterNew(2, a, b) * * cdef inline object PyArray_MultiIterNew3(a, b, c): # <<<<<<<<<<<<<< @@ -3903,7 +4141,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew3(PyObject *__ return __pyx_r; } -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":830 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":830 * return PyArray_MultiIterNew(3, a, b, c) * * cdef inline object PyArray_MultiIterNew4(a, b, c, d): # <<<<<<<<<<<<<< @@ -3917,7 +4155,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew4(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew4", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":831 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":831 * * cdef inline object PyArray_MultiIterNew4(a, b, c, d): * return PyArray_MultiIterNew(4, a, b, c, d) # <<<<<<<<<<<<<< @@ -3931,7 +4169,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew4(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":830 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":830 * return PyArray_MultiIterNew(3, a, b, c) * * cdef inline object PyArray_MultiIterNew4(a, b, c, d): # <<<<<<<<<<<<<< @@ -3950,7 +4188,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew4(PyObject *__ return __pyx_r; } -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":833 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":833 * return PyArray_MultiIterNew(4, a, b, c, d) * * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): # <<<<<<<<<<<<<< @@ -3964,7 +4202,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew5(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew5", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":834 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":834 * * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): * return PyArray_MultiIterNew(5, a, b, c, d, e) # <<<<<<<<<<<<<< @@ -3978,7 +4216,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew5(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":833 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":833 * return PyArray_MultiIterNew(4, a, b, c, d) * * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): # <<<<<<<<<<<<<< @@ -3997,7 +4235,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew5(PyObject *__ return __pyx_r; } -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":836 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":836 * return PyArray_MultiIterNew(5, a, b, c, d, e) * * cdef inline tuple PyDataType_SHAPE(dtype d): # <<<<<<<<<<<<<< @@ -4011,7 +4249,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ int __pyx_t_1; __Pyx_RefNannySetupContext("PyDataType_SHAPE", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":837 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":837 * * cdef inline tuple PyDataType_SHAPE(dtype d): * if PyDataType_HASSUBARRAY(d): # <<<<<<<<<<<<<< @@ -4021,7 +4259,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ __pyx_t_1 = (PyDataType_HASSUBARRAY(__pyx_v_d) != 0); if (__pyx_t_1) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":838 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":838 * cdef inline tuple PyDataType_SHAPE(dtype d): * if PyDataType_HASSUBARRAY(d): * return d.subarray.shape # <<<<<<<<<<<<<< @@ -4033,7 +4271,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ __pyx_r = ((PyObject*)__pyx_v_d->subarray->shape); goto __pyx_L0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":837 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":837 * * cdef inline tuple PyDataType_SHAPE(dtype d): * if PyDataType_HASSUBARRAY(d): # <<<<<<<<<<<<<< @@ -4042,7 +4280,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ */ } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":840 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":840 * return d.subarray.shape * else: * return () # <<<<<<<<<<<<<< @@ -4056,7 +4294,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ goto __pyx_L0; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":836 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":836 * return PyArray_MultiIterNew(5, a, b, c, d, e) * * cdef inline tuple PyDataType_SHAPE(dtype d): # <<<<<<<<<<<<<< @@ -4071,7 +4309,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ return __pyx_r; } -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":842 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":842 * return () * * cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset) except NULL: # <<<<<<<<<<<<<< @@ -4100,7 +4338,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx char *__pyx_t_9; __Pyx_RefNannySetupContext("_util_dtypestring", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":847 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":847 * * cdef dtype child * cdef int endian_detector = 1 # <<<<<<<<<<<<<< @@ -4109,7 +4347,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ __pyx_v_endian_detector = 1; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":848 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":848 * cdef dtype child * cdef int endian_detector = 1 * cdef bint little_endian = ((&endian_detector)[0] != 0) # <<<<<<<<<<<<<< @@ -4118,7 +4356,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ __pyx_v_little_endian = ((((char *)(&__pyx_v_endian_detector))[0]) != 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":851 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":851 * cdef tuple fields * * for childname in descr.names: # <<<<<<<<<<<<<< @@ -4141,7 +4379,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_XDECREF_SET(__pyx_v_childname, __pyx_t_3); __pyx_t_3 = 0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":852 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":852 * * for childname in descr.names: * fields = descr.fields[childname] # <<<<<<<<<<<<<< @@ -4158,7 +4396,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_XDECREF_SET(__pyx_v_fields, ((PyObject*)__pyx_t_3)); __pyx_t_3 = 0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":853 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":853 * for childname in descr.names: * fields = descr.fields[childname] * child, new_offset = fields # <<<<<<<<<<<<<< @@ -4193,7 +4431,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_XDECREF_SET(__pyx_v_new_offset, __pyx_t_4); __pyx_t_4 = 0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":855 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":855 * child, new_offset = fields * * if (end - f) - (new_offset - offset[0]) < 15: # <<<<<<<<<<<<<< @@ -4210,7 +4448,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_6 = ((((__pyx_v_end - __pyx_v_f) - ((int)__pyx_t_5)) < 15) != 0); if (unlikely(__pyx_t_6)) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":856 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":856 * * if (end - f) - (new_offset - offset[0]) < 15: * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") # <<<<<<<<<<<<<< @@ -4223,7 +4461,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 856, __pyx_L1_error) - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":855 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":855 * child, new_offset = fields * * if (end - f) - (new_offset - offset[0]) < 15: # <<<<<<<<<<<<<< @@ -4232,7 +4470,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * * if ((child.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -4252,7 +4490,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx } __pyx_L8_next_or:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":859 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":859 * * if ((child.byteorder == c'>' and little_endian) or * (child.byteorder == c'<' and not little_endian)): # <<<<<<<<<<<<<< @@ -4269,7 +4507,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_6 = __pyx_t_7; __pyx_L7_bool_binop_done:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * * if ((child.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -4278,7 +4516,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ if (unlikely(__pyx_t_6)) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":860 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":860 * if ((child.byteorder == c'>' and little_endian) or * (child.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< @@ -4291,7 +4529,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 860, __pyx_L1_error) - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * * if ((child.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -4300,7 +4538,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":870 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":870 * * # Output padding bytes * while offset[0] < new_offset: # <<<<<<<<<<<<<< @@ -4316,7 +4554,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (!__pyx_t_6) break; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":871 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":871 * # Output padding bytes * while offset[0] < new_offset: * f[0] = 120 # "x"; pad byte # <<<<<<<<<<<<<< @@ -4325,7 +4563,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ (__pyx_v_f[0]) = 0x78; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":872 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":872 * while offset[0] < new_offset: * f[0] = 120 # "x"; pad byte * f += 1 # <<<<<<<<<<<<<< @@ -4334,7 +4572,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ __pyx_v_f = (__pyx_v_f + 1); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":873 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":873 * f[0] = 120 # "x"; pad byte * f += 1 * offset[0] += 1 # <<<<<<<<<<<<<< @@ -4345,7 +4583,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx (__pyx_v_offset[__pyx_t_8]) = ((__pyx_v_offset[__pyx_t_8]) + 1); } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":875 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":875 * offset[0] += 1 * * offset[0] += child.itemsize # <<<<<<<<<<<<<< @@ -4355,7 +4593,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_8 = 0; (__pyx_v_offset[__pyx_t_8]) = ((__pyx_v_offset[__pyx_t_8]) + __pyx_v_child->elsize); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":877 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":877 * offset[0] += child.itemsize * * if not PyDataType_HASFIELDS(child): # <<<<<<<<<<<<<< @@ -4365,7 +4603,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_6 = ((!(PyDataType_HASFIELDS(__pyx_v_child) != 0)) != 0); if (__pyx_t_6) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":878 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":878 * * if not PyDataType_HASFIELDS(child): * t = child.type_num # <<<<<<<<<<<<<< @@ -4377,7 +4615,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_XDECREF_SET(__pyx_v_t, __pyx_t_4); __pyx_t_4 = 0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":879 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":879 * if not PyDataType_HASFIELDS(child): * t = child.type_num * if end - f < 5: # <<<<<<<<<<<<<< @@ -4387,7 +4625,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_6 = (((__pyx_v_end - __pyx_v_f) < 5) != 0); if (unlikely(__pyx_t_6)) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":880 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":880 * t = child.type_num * if end - f < 5: * raise RuntimeError(u"Format string allocated too short.") # <<<<<<<<<<<<<< @@ -4400,7 +4638,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __PYX_ERR(1, 880, __pyx_L1_error) - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":879 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":879 * if not PyDataType_HASFIELDS(child): * t = child.type_num * if end - f < 5: # <<<<<<<<<<<<<< @@ -4409,7 +4647,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":883 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":883 * * # Until ticket #99 is fixed, use integers to avoid warnings * if t == NPY_BYTE: f[0] = 98 #"b" # <<<<<<<<<<<<<< @@ -4427,7 +4665,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":884 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":884 * # Until ticket #99 is fixed, use integers to avoid warnings * if t == NPY_BYTE: f[0] = 98 #"b" * elif t == NPY_UBYTE: f[0] = 66 #"B" # <<<<<<<<<<<<<< @@ -4445,7 +4683,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":885 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":885 * if t == NPY_BYTE: f[0] = 98 #"b" * elif t == NPY_UBYTE: f[0] = 66 #"B" * elif t == NPY_SHORT: f[0] = 104 #"h" # <<<<<<<<<<<<<< @@ -4463,7 +4701,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":886 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":886 * elif t == NPY_UBYTE: f[0] = 66 #"B" * elif t == NPY_SHORT: f[0] = 104 #"h" * elif t == NPY_USHORT: f[0] = 72 #"H" # <<<<<<<<<<<<<< @@ -4481,7 +4719,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":887 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":887 * elif t == NPY_SHORT: f[0] = 104 #"h" * elif t == NPY_USHORT: f[0] = 72 #"H" * elif t == NPY_INT: f[0] = 105 #"i" # <<<<<<<<<<<<<< @@ -4499,7 +4737,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":888 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":888 * elif t == NPY_USHORT: f[0] = 72 #"H" * elif t == NPY_INT: f[0] = 105 #"i" * elif t == NPY_UINT: f[0] = 73 #"I" # <<<<<<<<<<<<<< @@ -4517,7 +4755,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":889 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":889 * elif t == NPY_INT: f[0] = 105 #"i" * elif t == NPY_UINT: f[0] = 73 #"I" * elif t == NPY_LONG: f[0] = 108 #"l" # <<<<<<<<<<<<<< @@ -4535,7 +4773,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":890 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":890 * elif t == NPY_UINT: f[0] = 73 #"I" * elif t == NPY_LONG: f[0] = 108 #"l" * elif t == NPY_ULONG: f[0] = 76 #"L" # <<<<<<<<<<<<<< @@ -4553,7 +4791,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":891 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":891 * elif t == NPY_LONG: f[0] = 108 #"l" * elif t == NPY_ULONG: f[0] = 76 #"L" * elif t == NPY_LONGLONG: f[0] = 113 #"q" # <<<<<<<<<<<<<< @@ -4571,7 +4809,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":892 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":892 * elif t == NPY_ULONG: f[0] = 76 #"L" * elif t == NPY_LONGLONG: f[0] = 113 #"q" * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" # <<<<<<<<<<<<<< @@ -4589,7 +4827,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":893 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":893 * elif t == NPY_LONGLONG: f[0] = 113 #"q" * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" * elif t == NPY_FLOAT: f[0] = 102 #"f" # <<<<<<<<<<<<<< @@ -4607,7 +4845,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":894 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":894 * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" * elif t == NPY_FLOAT: f[0] = 102 #"f" * elif t == NPY_DOUBLE: f[0] = 100 #"d" # <<<<<<<<<<<<<< @@ -4625,7 +4863,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":895 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":895 * elif t == NPY_FLOAT: f[0] = 102 #"f" * elif t == NPY_DOUBLE: f[0] = 100 #"d" * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" # <<<<<<<<<<<<<< @@ -4643,7 +4881,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":896 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":896 * elif t == NPY_DOUBLE: f[0] = 100 #"d" * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf # <<<<<<<<<<<<<< @@ -4663,7 +4901,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":897 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":897 * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd # <<<<<<<<<<<<<< @@ -4683,7 +4921,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":898 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":898 * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd * elif t == NPY_CLONGDOUBLE: f[0] = 90; f[1] = 103; f += 1 # Zg # <<<<<<<<<<<<<< @@ -4703,7 +4941,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":899 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":899 * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd * elif t == NPY_CLONGDOUBLE: f[0] = 90; f[1] = 103; f += 1 # Zg * elif t == NPY_OBJECT: f[0] = 79 #"O" # <<<<<<<<<<<<<< @@ -4721,7 +4959,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":901 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":901 * elif t == NPY_OBJECT: f[0] = 79 #"O" * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) # <<<<<<<<<<<<<< @@ -4740,7 +4978,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx } __pyx_L15:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":902 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":902 * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) * f += 1 # <<<<<<<<<<<<<< @@ -4749,7 +4987,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ __pyx_v_f = (__pyx_v_f + 1); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":877 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":877 * offset[0] += child.itemsize * * if not PyDataType_HASFIELDS(child): # <<<<<<<<<<<<<< @@ -4759,7 +4997,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L13; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":906 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":906 * # Cython ignores struct boundary information ("T{...}"), * # so don't output it * f = _util_dtypestring(child, f, end, offset) # <<<<<<<<<<<<<< @@ -4772,7 +5010,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx } __pyx_L13:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":851 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":851 * cdef tuple fields * * for childname in descr.names: # <<<<<<<<<<<<<< @@ -4782,7 +5020,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":907 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":907 * # so don't output it * f = _util_dtypestring(child, f, end, offset) * return f # <<<<<<<<<<<<<< @@ -4792,7 +5030,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_r = __pyx_v_f; goto __pyx_L0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":842 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":842 * return () * * cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset) except NULL: # <<<<<<<<<<<<<< @@ -4817,7 +5055,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx return __pyx_r; } -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1022 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1022 * int _import_umath() except -1 * * cdef inline void set_array_base(ndarray arr, object base): # <<<<<<<<<<<<<< @@ -4829,7 +5067,7 @@ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_a __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("set_array_base", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1023 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1023 * * cdef inline void set_array_base(ndarray arr, object base): * Py_INCREF(base) # important to do this before stealing the reference below! # <<<<<<<<<<<<<< @@ -4838,7 +5076,7 @@ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_a */ Py_INCREF(__pyx_v_base); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1024 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1024 * cdef inline void set_array_base(ndarray arr, object base): * Py_INCREF(base) # important to do this before stealing the reference below! * PyArray_SetBaseObject(arr, base) # <<<<<<<<<<<<<< @@ -4847,7 +5085,7 @@ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_a */ (void)(PyArray_SetBaseObject(__pyx_v_arr, __pyx_v_base)); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1022 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1022 * int _import_umath() except -1 * * cdef inline void set_array_base(ndarray arr, object base): # <<<<<<<<<<<<<< @@ -4859,7 +5097,7 @@ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_a __Pyx_RefNannyFinishContext(); } -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1026 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1026 * PyArray_SetBaseObject(arr, base) * * cdef inline object get_array_base(ndarray arr): # <<<<<<<<<<<<<< @@ -4874,7 +5112,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py int __pyx_t_1; __Pyx_RefNannySetupContext("get_array_base", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1027 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1027 * * cdef inline object get_array_base(ndarray arr): * base = PyArray_BASE(arr) # <<<<<<<<<<<<<< @@ -4883,7 +5121,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py */ __pyx_v_base = PyArray_BASE(__pyx_v_arr); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1028 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1028 * cdef inline object get_array_base(ndarray arr): * base = PyArray_BASE(arr) * if base is NULL: # <<<<<<<<<<<<<< @@ -4893,7 +5131,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py __pyx_t_1 = ((__pyx_v_base == NULL) != 0); if (__pyx_t_1) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1029 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1029 * base = PyArray_BASE(arr) * if base is NULL: * return None # <<<<<<<<<<<<<< @@ -4904,7 +5142,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1028 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1028 * cdef inline object get_array_base(ndarray arr): * base = PyArray_BASE(arr) * if base is NULL: # <<<<<<<<<<<<<< @@ -4913,7 +5151,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py */ } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1030 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1030 * if base is NULL: * return None * return base # <<<<<<<<<<<<<< @@ -4925,7 +5163,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py __pyx_r = ((PyObject *)__pyx_v_base); goto __pyx_L0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1026 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1026 * PyArray_SetBaseObject(arr, base) * * cdef inline object get_array_base(ndarray arr): # <<<<<<<<<<<<<< @@ -4940,7 +5178,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py return __pyx_r; } -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1034 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1034 * # Versions of the import_* functions which are more suitable for * # Cython code. * cdef inline int import_array() except -1: # <<<<<<<<<<<<<< @@ -4961,7 +5199,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { PyObject *__pyx_t_8 = NULL; __Pyx_RefNannySetupContext("import_array", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 * # Cython code. * cdef inline int import_array() except -1: * try: # <<<<<<<<<<<<<< @@ -4977,7 +5215,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { __Pyx_XGOTREF(__pyx_t_3); /*try:*/ { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1036 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1036 * cdef inline int import_array() except -1: * try: * _import_array() # <<<<<<<<<<<<<< @@ -4986,7 +5224,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { */ __pyx_t_4 = _import_array(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 1036, __pyx_L3_error) - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 * # Cython code. * cdef inline int import_array() except -1: * try: # <<<<<<<<<<<<<< @@ -5000,7 +5238,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { goto __pyx_L8_try_end; __pyx_L3_error:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1037 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1037 * try: * _import_array() * except Exception: # <<<<<<<<<<<<<< @@ -5015,7 +5253,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { __Pyx_GOTREF(__pyx_t_6); __Pyx_GOTREF(__pyx_t_7); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1038 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1038 * _import_array() * except Exception: * raise ImportError("numpy.core.multiarray failed to import") # <<<<<<<<<<<<<< @@ -5031,7 +5269,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 * # Cython code. * cdef inline int import_array() except -1: * try: # <<<<<<<<<<<<<< @@ -5046,7 +5284,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { __pyx_L8_try_end:; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1034 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1034 * # Versions of the import_* functions which are more suitable for * # Cython code. * cdef inline int import_array() except -1: # <<<<<<<<<<<<<< @@ -5069,7 +5307,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { return __pyx_r; } -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1040 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1040 * raise ImportError("numpy.core.multiarray failed to import") * * cdef inline int import_umath() except -1: # <<<<<<<<<<<<<< @@ -5090,7 +5328,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { PyObject *__pyx_t_8 = NULL; __Pyx_RefNannySetupContext("import_umath", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 * * cdef inline int import_umath() except -1: * try: # <<<<<<<<<<<<<< @@ -5106,7 +5344,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { __Pyx_XGOTREF(__pyx_t_3); /*try:*/ { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1042 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1042 * cdef inline int import_umath() except -1: * try: * _import_umath() # <<<<<<<<<<<<<< @@ -5115,7 +5353,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { */ __pyx_t_4 = _import_umath(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 1042, __pyx_L3_error) - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 * * cdef inline int import_umath() except -1: * try: # <<<<<<<<<<<<<< @@ -5129,7 +5367,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { goto __pyx_L8_try_end; __pyx_L3_error:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1043 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1043 * try: * _import_umath() * except Exception: # <<<<<<<<<<<<<< @@ -5144,7 +5382,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { __Pyx_GOTREF(__pyx_t_6); __Pyx_GOTREF(__pyx_t_7); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1044 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1044 * _import_umath() * except Exception: * raise ImportError("numpy.core.umath failed to import") # <<<<<<<<<<<<<< @@ -5160,7 +5398,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 * * cdef inline int import_umath() except -1: * try: # <<<<<<<<<<<<<< @@ -5175,7 +5413,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { __pyx_L8_try_end:; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1040 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1040 * raise ImportError("numpy.core.multiarray failed to import") * * cdef inline int import_umath() except -1: # <<<<<<<<<<<<<< @@ -5198,7 +5436,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { return __pyx_r; } -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 +/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 * raise ImportError("numpy.core.umath failed to import") * * cdef inline int import_ufunc() except -1: # <<<<<<<<<<<<<< @@ -5219,7 +5457,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { PyObject *__pyx_t_8 = NULL; __Pyx_RefNannySetupContext("import_ufunc", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 * * cdef inline int import_ufunc() except -1: * try: # <<<<<<<<<<<<<< @@ -5235,7 +5473,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { __Pyx_XGOTREF(__pyx_t_3); /*try:*/ { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1048 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1048 * cdef inline int import_ufunc() except -1: * try: * _import_umath() # <<<<<<<<<<<<<< @@ -5244,7 +5482,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { */ __pyx_t_4 = _import_umath(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 1048, __pyx_L3_error) - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 * * cdef inline int import_ufunc() except -1: * try: # <<<<<<<<<<<<<< @@ -5258,7 +5496,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { goto __pyx_L8_try_end; __pyx_L3_error:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1049 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1049 * try: * _import_umath() * except Exception: # <<<<<<<<<<<<<< @@ -5272,7 +5510,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { __Pyx_GOTREF(__pyx_t_6); __Pyx_GOTREF(__pyx_t_7); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1050 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1050 * _import_umath() * except Exception: * raise ImportError("numpy.core.umath failed to import") # <<<<<<<<<<<<<< @@ -5286,7 +5524,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 * * cdef inline int import_ufunc() except -1: * try: # <<<<<<<<<<<<<< @@ -5301,7 +5539,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { __pyx_L8_try_end:; } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 * raise ImportError("numpy.core.umath failed to import") * * cdef inline int import_ufunc() except -1: # <<<<<<<<<<<<<< @@ -5324,6 +5562,224 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { return __pyx_r; } +/* "cfunc.to_py":65 + * @cname("__Pyx_CFunc_object____object___to_py") + * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): + * def wrap(object b): # <<<<<<<<<<<<<< + * """wrap(b)""" + * return f(b) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap(PyObject *__pyx_self, PyObject *__pyx_v_b); /*proto*/ +static char __pyx_doc_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap[] = "wrap(b)"; +static PyMethodDef __pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap = {"wrap", (PyCFunction)__pyx_pw_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, METH_O, __pyx_doc_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap}; +static PyObject *__pyx_pw_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap(PyObject *__pyx_self, PyObject *__pyx_v_b) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("wrap (wrapper)", 0); + __pyx_r = __pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(__pyx_self, ((PyObject *)__pyx_v_b)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(PyObject *__pyx_self, PyObject *__pyx_v_b) { + struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_cur_scope; + struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("wrap", 0); + __pyx_outer_scope = (struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + + /* "cfunc.to_py":67 + * def wrap(object b): + * """wrap(b)""" + * return f(b) # <<<<<<<<<<<<<< + * return wrap + * + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_cur_scope->__pyx_v_f(__pyx_v_b); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 67, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "cfunc.to_py":65 + * @cname("__Pyx_CFunc_object____object___to_py") + * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): + * def wrap(object b): # <<<<<<<<<<<<<< + * """wrap(b)""" + * return f(b) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("cfunc.to_py.__Pyx_CFunc_object____object___to_py.wrap", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cfunc.to_py":64 + * + * @cname("__Pyx_CFunc_object____object___to_py") + * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< + * def wrap(object b): + * """wrap(b)""" + */ + +static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*__pyx_v_f)(PyObject *)) { + struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_cur_scope; + PyObject *__pyx_v_wrap = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__Pyx_CFunc_object____object___to_py", 0); + __pyx_cur_scope = (struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(__pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py, __pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(2, 64, __pyx_L1_error) + } else { + __Pyx_GOTREF(__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_v_f = __pyx_v_f; + + /* "cfunc.to_py":65 + * @cname("__Pyx_CFunc_object____object___to_py") + * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): + * def wrap(object b): # <<<<<<<<<<<<<< + * """wrap(b)""" + * return f(b) + */ + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, 0, __pyx_n_s_Pyx_CFunc_object____object___t, ((PyObject*)__pyx_cur_scope), __pyx_n_s_cfunc_to_py, __pyx_d, ((PyObject *)__pyx_codeobj__11)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 65, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_wrap = __pyx_t_1; + __pyx_t_1 = 0; + + /* "cfunc.to_py":68 + * """wrap(b)""" + * return f(b) + * return wrap # <<<<<<<<<<<<<< + * + * + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_wrap); + __pyx_r = __pyx_v_wrap; + goto __pyx_L0; + + /* "cfunc.to_py":64 + * + * @cname("__Pyx_CFunc_object____object___to_py") + * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< + * def wrap(object b): + * """wrap(b)""" + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("cfunc.to_py.__Pyx_CFunc_object____object___to_py", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_wrap); + __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[8]; +static int __pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py = 0; + +static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + PyObject *o; + if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)))) { + o = (PyObject*)__pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[--__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py]; + memset(o, 0, sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)); + (void) PyObject_INIT(o, t); + } else { + o = (*t->tp_alloc)(t, 0); + if (unlikely(!o)) return 0; + } + return o; +} + +static void __pyx_tp_dealloc___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyObject *o) { + if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)))) { + __pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py++] = ((struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)o); + } else { + (*Py_TYPE(o)->tp_free)(o); + } +} + +static PyTypeObject __pyx_scope_struct____Pyx_CFunc_object____object___to_py = { + PyVarObject_HEAD_INIT(0, 0) + "gensim.models._utils_any2vec.__pyx_scope_struct____Pyx_CFunc_object____object___to_py", /*tp_name*/ + sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc___pyx_scope_struct____Pyx_CFunc_object____object___to_py, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + static PyMethodDef __pyx_methods[] = { {"ft_hash_bytes", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash_bytes, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_ft_hash_bytes}, {"ft_hash_broken", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_2ft_hash_broken}, @@ -5382,12 +5838,16 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_MB_START, __pyx_k_MB_START, sizeof(__pyx_k_MB_START), 0, 0, 1, 1}, {&__pyx_kp_u_Non_native_byte_order_not_suppor, __pyx_k_Non_native_byte_order_not_suppor, sizeof(__pyx_k_Non_native_byte_order_not_suppor), 0, 1, 0, 0}, {&__pyx_n_s_PY2, __pyx_k_PY2, sizeof(__pyx_k_PY2), 0, 0, 1, 1}, + {&__pyx_n_s_Pyx_CFunc_object____object___t, __pyx_k_Pyx_CFunc_object____object___t, sizeof(__pyx_k_Pyx_CFunc_object____object___t), 0, 0, 1, 1}, {&__pyx_n_s_RuntimeError, __pyx_k_RuntimeError, sizeof(__pyx_k_RuntimeError), 0, 0, 1, 1}, {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, {&__pyx_kp_u__2, __pyx_k__2, sizeof(__pyx_k__2), 0, 1, 0, 0}, + {&__pyx_n_s_b, __pyx_k_b, sizeof(__pyx_k_b), 0, 0, 1, 1}, + {&__pyx_n_s_cfunc_to_py, __pyx_k_cfunc_to_py, sizeof(__pyx_k_cfunc_to_py), 0, 0, 1, 1}, {&__pyx_n_s_cline_in_traceback, __pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 0, 1, 1}, {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, + {&__pyx_n_s_is_utf8_continue, __pyx_k_is_utf8_continue, sizeof(__pyx_k_is_utf8_continue), 0, 0, 1, 1}, {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, {&__pyx_n_s_max_n, __pyx_k_max_n, sizeof(__pyx_k_max_n), 0, 0, 1, 1}, {&__pyx_n_s_min_n, __pyx_k_min_n, sizeof(__pyx_k_min_n), 0, 0, 1, 1}, @@ -5401,10 +5861,12 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, {&__pyx_kp_s_s, __pyx_k_s, sizeof(__pyx_k_s), 0, 0, 1, 0}, {&__pyx_n_s_six, __pyx_k_six, sizeof(__pyx_k_six), 0, 0, 1, 1}, + {&__pyx_kp_s_stringsource, __pyx_k_stringsource, sizeof(__pyx_k_stringsource), 0, 0, 1, 0}, {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, {&__pyx_kp_u_unknown_dtype_code_in_numpy_pxd, __pyx_k_unknown_dtype_code_in_numpy_pxd, sizeof(__pyx_k_unknown_dtype_code_in_numpy_pxd), 0, 1, 0, 0}, {&__pyx_kp_s_utf_8, __pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 0, 1, 0}, {&__pyx_n_s_word, __pyx_k_word, sizeof(__pyx_k_word), 0, 0, 1, 1}, + {&__pyx_n_s_wrap, __pyx_k_wrap, sizeof(__pyx_k_wrap), 0, 0, 1, 1}, {0, 0, 0, 0, 0, 0, 0} }; static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(void) { @@ -5421,7 +5883,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): * raise ValueError(u"ndarray is not C contiguous") # <<<<<<<<<<<<<< @@ -5432,7 +5894,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__3); __Pyx_GIVEREF(__pyx_tuple__3); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): * raise ValueError(u"ndarray is not Fortran contiguous") # <<<<<<<<<<<<<< @@ -5443,7 +5905,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__4); __Pyx_GIVEREF(__pyx_tuple__4); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 * if ((descr.byteorder == c'>' and little_endian) or * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< @@ -5454,7 +5916,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__5); __Pyx_GIVEREF(__pyx_tuple__5); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":856 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":856 * * if (end - f) - (new_offset - offset[0]) < 15: * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") # <<<<<<<<<<<<<< @@ -5465,7 +5927,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__6); __Pyx_GIVEREF(__pyx_tuple__6); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":880 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":880 * t = child.type_num * if end - f < 5: * raise RuntimeError(u"Format string allocated too short.") # <<<<<<<<<<<<<< @@ -5476,7 +5938,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__7); __Pyx_GIVEREF(__pyx_tuple__7); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1038 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1038 * _import_array() * except Exception: * raise ImportError("numpy.core.multiarray failed to import") # <<<<<<<<<<<<<< @@ -5487,7 +5949,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__8); __Pyx_GIVEREF(__pyx_tuple__8); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1044 + /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1044 * _import_umath() * except Exception: * raise ImportError("numpy.core.umath failed to import") # <<<<<<<<<<<<<< @@ -5497,6 +5959,18 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __pyx_tuple__9 = PyTuple_Pack(1, __pyx_kp_s_numpy_core_umath_failed_to_impor); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(1, 1044, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__9); __Pyx_GIVEREF(__pyx_tuple__9); + + /* "cfunc.to_py":65 + * @cname("__Pyx_CFunc_object____object___to_py") + * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): + * def wrap(object b): # <<<<<<<<<<<<<< + * """wrap(b)""" + * return f(b) + */ + __pyx_tuple__10 = PyTuple_Pack(1, __pyx_n_s_b); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(2, 65, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__10); + __Pyx_GIVEREF(__pyx_tuple__10); + __pyx_codeobj__11 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__10, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_stringsource, __pyx_n_s_wrap, 65, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__11)) __PYX_ERR(2, 65, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -5551,8 +6025,17 @@ static int __Pyx_modinit_type_init_code(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); /*--- Type init code ---*/ + if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(2, 64, __pyx_L1_error) + __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_print = 0; + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_dictoffset && __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_getattro == PyObject_GenericGetAttr)) { + __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; + } + __pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py = &__pyx_scope_struct____Pyx_CFunc_object____object___to_py; __Pyx_RefNannyFinishContext(); return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; } static int __Pyx_modinit_type_import_code(void) { @@ -5560,7 +6043,7 @@ static int __Pyx_modinit_type_import_code(void) { PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); /*--- Type import code ---*/ - __pyx_t_1 = PyImport_ImportModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 9, __pyx_L1_error) + __pyx_t_1 = PyImport_ImportModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 9, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_ptype_7cpython_4type_type = __Pyx_ImportType(__pyx_t_1, __Pyx_BUILTIN_MODULE_NAME, "type", #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 @@ -5569,7 +6052,7 @@ static int __Pyx_modinit_type_import_code(void) { sizeof(PyHeapTypeObject), #endif __Pyx_ImportType_CheckSize_Warn); - if (!__pyx_ptype_7cpython_4type_type) __PYX_ERR(2, 9, __pyx_L1_error) + if (!__pyx_ptype_7cpython_4type_type) __PYX_ERR(3, 9, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = PyImport_ImportModule("numpy"); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 206, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); @@ -5702,6 +6185,7 @@ static CYTHON_SMALL_CODE int __pyx_pymod_exec__utils_any2vec(PyObject *__pyx_pyi { PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; __Pyx_RefNannyDeclarations #if CYTHON_PEP489_MULTI_PHASE_INIT if (__pyx_m) { @@ -5798,7 +6282,7 @@ if (!__Pyx_RefNanny) { (void)__Pyx_modinit_global_init_code(); (void)__Pyx_modinit_variable_export_code(); (void)__Pyx_modinit_function_export_code(); - (void)__Pyx_modinit_type_init_code(); + if (unlikely(__Pyx_modinit_type_init_code() != 0)) goto __pyx_L1_error; if (unlikely(__Pyx_modinit_type_import_code() != 0)) goto __pyx_L1_error; (void)__Pyx_modinit_variable_import_code(); (void)__Pyx_modinit_function_import_code(); @@ -5858,6 +6342,31 @@ if (!__Pyx_RefNanny) { */ if (PyDict_SetItem(__pyx_d, __pyx_n_s_MB_START, __pyx_int_128) < 0) __PYX_ERR(0, 101, __pyx_L1_error) + /* "gensim/models/_utils_any2vec.pyx":112 + * + * + * _is_utf8_continue = _is_utf8_continue_py2 if PY2 else _is_utf8_continue_py3 # <<<<<<<<<<<<<< + * + * + */ + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 112, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 112, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_3) { + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__is_utf8_continue_py2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 112, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_t_1; + __pyx_t_1 = 0; + } else { + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__is_utf8_continue_py3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 112, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_t_1; + __pyx_t_1 = 0; + } + if (PyDict_SetItem(__pyx_d, __pyx_n_s_is_utf8_continue, __pyx_t_2) < 0) __PYX_ERR(0, 112, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + /* "gensim/models/_utils_any2vec.pyx":1 * #!/usr/bin/env cython # <<<<<<<<<<<<<< * # cython: boundscheck=False @@ -5868,12 +6377,12 @@ if (!__Pyx_RefNanny) { if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_2) < 0) __PYX_ERR(0, 1, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 - * raise ImportError("numpy.core.umath failed to import") + /* "cfunc.to_py":64 * - * cdef inline int import_ufunc() except -1: # <<<<<<<<<<<<<< - * try: - * _import_umath() + * @cname("__Pyx_CFunc_object____object___to_py") + * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< + * def wrap(object b): + * """wrap(b)""" */ /*--- Wrapped vars code ---*/ @@ -6416,6 +6925,41 @@ static int __Pyx_ParseOptionalKeywords( return -1; } +/* GetModuleGlobalName */ +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ + PyObject *result; +#if !CYTHON_AVOID_BORROWED_REFS +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 + result = _PyDict_GetItem_KnownHash(__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } else if (unlikely(PyErr_Occurred())) { + return NULL; + } +#else + result = PyDict_GetItem(__pyx_d, name); + __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } +#endif +#else + result = PyObject_GetItem(__pyx_d, name); + __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); +} + /* PyCFunctionFastCall */ #if CYTHON_FAST_PYCCALL static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { @@ -6734,41 +7278,6 @@ static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); } -/* GetModuleGlobalName */ -#if CYTHON_USE_DICT_VERSIONS -static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) -#else -static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) -#endif -{ - PyObject *result; -#if !CYTHON_AVOID_BORROWED_REFS -#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 - result = _PyDict_GetItem_KnownHash(__pyx_d, name, ((PyASCIIObject *) name)->hash); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } else if (unlikely(PyErr_Occurred())) { - return NULL; - } -#else - result = PyDict_GetItem(__pyx_d, name); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } -#endif -#else - result = PyObject_GetItem(__pyx_d, name); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } - PyErr_Clear(); -#endif - return __Pyx_GetBuiltinName(name); -} - /* PyErrFetchRestore */ #if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { @@ -7162,6 +7671,681 @@ static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) return -1; } +/* FetchCommonType */ +static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { + PyObject* fake_module; + PyTypeObject* cached_type = NULL; + fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); + if (!fake_module) return NULL; + Py_INCREF(fake_module); + cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); + if (cached_type) { + if (!PyType_Check((PyObject*)cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", + type->tp_name); + goto bad; + } + if (cached_type->tp_basicsize != type->tp_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + type->tp_name); + goto bad; + } + } else { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + PyErr_Clear(); + if (PyType_Ready(type) < 0) goto bad; + if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) + goto bad; + Py_INCREF(type); + cached_type = type; + } +done: + Py_DECREF(fake_module); + return cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +/* CythonFunction */ +#include +static PyObject * +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure) +{ + if (unlikely(op->func_doc == NULL)) { + if (op->func.m_ml->ml_doc) { +#if PY_MAJOR_VERSION >= 3 + op->func_doc = PyUnicode_FromString(op->func.m_ml->ml_doc); +#else + op->func_doc = PyString_FromString(op->func.m_ml->ml_doc); +#endif + if (unlikely(op->func_doc == NULL)) + return NULL; + } else { + Py_INCREF(Py_None); + return Py_None; + } + } + Py_INCREF(op->func_doc); + return op->func_doc; +} +static int +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) +{ + PyObject *tmp = op->func_doc; + if (value == NULL) { + value = Py_None; + } + Py_INCREF(value); + op->func_doc = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +{ + if (unlikely(op->func_name == NULL)) { +#if PY_MAJOR_VERSION >= 3 + op->func_name = PyUnicode_InternFromString(op->func.m_ml->ml_name); +#else + op->func_name = PyString_InternFromString(op->func.m_ml->ml_name); +#endif + if (unlikely(op->func_name == NULL)) + return NULL; + } + Py_INCREF(op->func_name); + return op->func_name; +} +static int +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) +{ + PyObject *tmp; +#if PY_MAJOR_VERSION >= 3 + if (unlikely(value == NULL || !PyUnicode_Check(value))) +#else + if (unlikely(value == NULL || !PyString_Check(value))) +#endif + { + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + tmp = op->func_name; + Py_INCREF(value); + op->func_name = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +{ + Py_INCREF(op->func_qualname); + return op->func_qualname; +} +static int +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) +{ + PyObject *tmp; +#if PY_MAJOR_VERSION >= 3 + if (unlikely(value == NULL || !PyUnicode_Check(value))) +#else + if (unlikely(value == NULL || !PyString_Check(value))) +#endif + { + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + tmp = op->func_qualname; + Py_INCREF(value); + op->func_qualname = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_self(__pyx_CyFunctionObject *m, CYTHON_UNUSED void *closure) +{ + PyObject *self; + self = m->func_closure; + if (self == NULL) + self = Py_None; + Py_INCREF(self); + return self; +} +static PyObject * +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +{ + if (unlikely(op->func_dict == NULL)) { + op->func_dict = PyDict_New(); + if (unlikely(op->func_dict == NULL)) + return NULL; + } + Py_INCREF(op->func_dict); + return op->func_dict; +} +static int +__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) +{ + PyObject *tmp; + if (unlikely(value == NULL)) { + PyErr_SetString(PyExc_TypeError, + "function's dictionary may not be deleted"); + return -1; + } + if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "setting function's dictionary to a non-dict"); + return -1; + } + tmp = op->func_dict; + Py_INCREF(value); + op->func_dict = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +{ + Py_INCREF(op->func_globals); + return op->func_globals; +} +static PyObject * +__Pyx_CyFunction_get_closure(CYTHON_UNUSED __pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +{ + Py_INCREF(Py_None); + return Py_None; +} +static PyObject * +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +{ + PyObject* result = (op->func_code) ? op->func_code : Py_None; + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { + int result = 0; + PyObject *res = op->defaults_getter((PyObject *) op); + if (unlikely(!res)) + return -1; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + op->defaults_tuple = PyTuple_GET_ITEM(res, 0); + Py_INCREF(op->defaults_tuple); + op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); + Py_INCREF(op->defaults_kwdict); + #else + op->defaults_tuple = PySequence_ITEM(res, 0); + if (unlikely(!op->defaults_tuple)) result = -1; + else { + op->defaults_kwdict = PySequence_ITEM(res, 1); + if (unlikely(!op->defaults_kwdict)) result = -1; + } + #endif + Py_DECREF(res); + return result; +} +static int +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { + PyObject* tmp; + if (!value) { + value = Py_None; + } else if (value != Py_None && !PyTuple_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "__defaults__ must be set to a tuple object"); + return -1; + } + Py_INCREF(value); + tmp = op->defaults_tuple; + op->defaults_tuple = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { + PyObject* result = op->defaults_tuple; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; + result = op->defaults_tuple; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { + PyObject* tmp; + if (!value) { + value = Py_None; + } else if (value != Py_None && !PyDict_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "__kwdefaults__ must be set to a dict object"); + return -1; + } + Py_INCREF(value); + tmp = op->defaults_kwdict; + op->defaults_kwdict = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { + PyObject* result = op->defaults_kwdict; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; + result = op->defaults_kwdict; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { + PyObject* tmp; + if (!value || value == Py_None) { + value = NULL; + } else if (!PyDict_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "__annotations__ must be set to a dict object"); + return -1; + } + Py_XINCREF(value); + tmp = op->func_annotations; + op->func_annotations = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { + PyObject* result = op->func_annotations; + if (unlikely(!result)) { + result = PyDict_New(); + if (unlikely(!result)) return NULL; + op->func_annotations = result; + } + Py_INCREF(result); + return result; +} +static PyGetSetDef __pyx_CyFunction_getsets[] = { + {(char *) "func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {(char *) "__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {(char *) "func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {(char *) "__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {(char *) "__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, + {(char *) "__self__", (getter)__Pyx_CyFunction_get_self, 0, 0, 0}, + {(char *) "func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {(char *) "__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {(char *) "func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {(char *) "__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {(char *) "func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {(char *) "__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {(char *) "func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {(char *) "__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {(char *) "func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {(char *) "__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {(char *) "__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, + {(char *) "__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {0, 0, 0, 0, 0} +}; +static PyMemberDef __pyx_CyFunction_members[] = { + {(char *) "__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), PY_WRITE_RESTRICTED, 0}, + {0, 0, 0, 0, 0} +}; +static PyObject * +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, CYTHON_UNUSED PyObject *args) +{ +#if PY_MAJOR_VERSION >= 3 + return PyUnicode_FromString(m->func.m_ml->ml_name); +#else + return PyString_FromString(m->func.m_ml->ml_name); +#endif +} +static PyMethodDef __pyx_CyFunction_methods[] = { + {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, + {0, 0, 0, 0} +}; +#if PY_VERSION_HEX < 0x030500A0 +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) +#else +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func.m_weakreflist) +#endif +static PyObject *__Pyx_CyFunction_New(PyTypeObject *type, PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + __pyx_CyFunctionObject *op = PyObject_GC_New(__pyx_CyFunctionObject, type); + if (op == NULL) + return NULL; + op->flags = flags; + __Pyx_CyFunction_weakreflist(op) = NULL; + op->func.m_ml = ml; + op->func.m_self = (PyObject *) op; + Py_XINCREF(closure); + op->func_closure = closure; + Py_XINCREF(module); + op->func.m_module = module; + op->func_dict = NULL; + op->func_name = NULL; + Py_INCREF(qualname); + op->func_qualname = qualname; + op->func_doc = NULL; + op->func_classobj = NULL; + op->func_globals = globals; + Py_INCREF(op->func_globals); + Py_XINCREF(code); + op->func_code = code; + op->defaults_pyobjects = 0; + op->defaults = NULL; + op->defaults_tuple = NULL; + op->defaults_kwdict = NULL; + op->defaults_getter = NULL; + op->func_annotations = NULL; + PyObject_GC_Track(op); + return (PyObject *) op; +} +static int +__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) +{ + Py_CLEAR(m->func_closure); + Py_CLEAR(m->func.m_module); + Py_CLEAR(m->func_dict); + Py_CLEAR(m->func_name); + Py_CLEAR(m->func_qualname); + Py_CLEAR(m->func_doc); + Py_CLEAR(m->func_globals); + Py_CLEAR(m->func_code); + Py_CLEAR(m->func_classobj); + Py_CLEAR(m->defaults_tuple); + Py_CLEAR(m->defaults_kwdict); + Py_CLEAR(m->func_annotations); + if (m->defaults) { + PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); + int i; + for (i = 0; i < m->defaults_pyobjects; i++) + Py_XDECREF(pydefaults[i]); + PyObject_Free(m->defaults); + m->defaults = NULL; + } + return 0; +} +static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + if (__Pyx_CyFunction_weakreflist(m) != NULL) + PyObject_ClearWeakRefs((PyObject *) m); + __Pyx_CyFunction_clear(m); + PyObject_GC_Del(m); +} +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + __Pyx__CyFunction_dealloc(m); +} +static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) +{ + Py_VISIT(m->func_closure); + Py_VISIT(m->func.m_module); + Py_VISIT(m->func_dict); + Py_VISIT(m->func_name); + Py_VISIT(m->func_qualname); + Py_VISIT(m->func_doc); + Py_VISIT(m->func_globals); + Py_VISIT(m->func_code); + Py_VISIT(m->func_classobj); + Py_VISIT(m->defaults_tuple); + Py_VISIT(m->defaults_kwdict); + if (m->defaults) { + PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); + int i; + for (i = 0; i < m->defaults_pyobjects; i++) + Py_VISIT(pydefaults[i]); + } + return 0; +} +static PyObject *__Pyx_CyFunction_descr_get(PyObject *func, PyObject *obj, PyObject *type) +{ + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + if (m->flags & __Pyx_CYFUNCTION_STATICMETHOD) { + Py_INCREF(func); + return func; + } + if (m->flags & __Pyx_CYFUNCTION_CLASSMETHOD) { + if (type == NULL) + type = (PyObject *)(Py_TYPE(obj)); + return __Pyx_PyMethod_New(func, type, (PyObject *)(Py_TYPE(type))); + } + if (obj == Py_None) + obj = NULL; + return __Pyx_PyMethod_New(func, obj, type); +} +static PyObject* +__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) +{ +#if PY_MAJOR_VERSION >= 3 + return PyUnicode_FromFormat("", + op->func_qualname, (void *)op); +#else + return PyString_FromFormat("", + PyString_AsString(op->func_qualname), (void *)op); +#endif +} +static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { + PyCFunctionObject* f = (PyCFunctionObject*)func; + PyCFunction meth = f->m_ml->ml_meth; + Py_ssize_t size; + switch (f->m_ml->ml_flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { + case METH_VARARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) + return (*meth)(self, arg); + break; + case METH_VARARGS | METH_KEYWORDS: + return (*(PyCFunctionWithKeywords)(void*)meth)(self, arg, kw); + case METH_NOARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { + size = PyTuple_GET_SIZE(arg); + if (likely(size == 0)) + return (*meth)(self, NULL); + PyErr_Format(PyExc_TypeError, + "%.200s() takes no arguments (%" CYTHON_FORMAT_SSIZE_T "d given)", + f->m_ml->ml_name, size); + return NULL; + } + break; + case METH_O: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { + size = PyTuple_GET_SIZE(arg); + if (likely(size == 1)) { + PyObject *result, *arg0; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + arg0 = PyTuple_GET_ITEM(arg, 0); + #else + arg0 = PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; + #endif + result = (*meth)(self, arg0); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(arg0); + #endif + return result; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes exactly one argument (%" CYTHON_FORMAT_SSIZE_T "d given)", + f->m_ml->ml_name, size); + return NULL; + } + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags in " + "__Pyx_CyFunction_Call. METH_OLDARGS is no " + "longer supported!"); + return NULL; + } + PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", + f->m_ml->ml_name); + return NULL; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + return __Pyx_CyFunction_CallMethod(func, ((PyCFunctionObject*)func)->m_self, arg, kw); +} +static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { + PyObject *result; + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + Py_ssize_t argc; + PyObject *new_args; + PyObject *self; + argc = PyTuple_GET_SIZE(args); + new_args = PyTuple_GetSlice(args, 1, argc); + if (unlikely(!new_args)) + return NULL; + self = PyTuple_GetItem(args, 0); + if (unlikely(!self)) { + Py_DECREF(new_args); + return NULL; + } + result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); + Py_DECREF(new_args); + } else { + result = __Pyx_CyFunction_Call(func, args, kw); + } + return result; +} +static PyTypeObject __pyx_CyFunctionType_type = { + PyVarObject_HEAD_INIT(0, 0) + "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, + (destructor) __Pyx_CyFunction_dealloc, + 0, + 0, + 0, +#if PY_MAJOR_VERSION < 3 + 0, +#else + 0, +#endif + (reprfunc) __Pyx_CyFunction_repr, + 0, + 0, + 0, + 0, + __Pyx_CyFunction_CallAsMethod, + 0, + 0, + 0, + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + 0, + (traverseproc) __Pyx_CyFunction_traverse, + (inquiry) __Pyx_CyFunction_clear, + 0, +#if PY_VERSION_HEX < 0x030500A0 + offsetof(__pyx_CyFunctionObject, func_weakreflist), +#else + offsetof(PyCFunctionObject, m_weakreflist), +#endif + 0, + 0, + __pyx_CyFunction_methods, + __pyx_CyFunction_members, + __pyx_CyFunction_getsets, + 0, + 0, + __Pyx_CyFunction_descr_get, + 0, + offsetof(__pyx_CyFunctionObject, func_dict), + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, +#if PY_VERSION_HEX >= 0x030400a1 + 0, +#endif +}; +static int __pyx_CyFunction_init(void) { + __pyx_CyFunctionType = __Pyx_FetchCommonType(&__pyx_CyFunctionType_type); + if (unlikely(__pyx_CyFunctionType == NULL)) { + return -1; + } + return 0; +} +static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *func, size_t size, int pyobjects) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults = PyObject_Malloc(size); + if (unlikely(!m->defaults)) + return PyErr_NoMemory(); + memset(m->defaults, 0, size); + m->defaults_pyobjects = pyobjects; + return m->defaults; +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_tuple = tuple; + Py_INCREF(tuple); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_kwdict = dict; + Py_INCREF(dict); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->func_annotations = dict; + Py_INCREF(dict); +} + +/* PyObject_GenericGetAttrNoDict */ +#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 +static PyObject *__Pyx_RaiseGenericGetAttributeError(PyTypeObject *tp, PyObject *attr_name) { + PyErr_Format(PyExc_AttributeError, +#if PY_MAJOR_VERSION >= 3 + "'%.50s' object has no attribute '%U'", + tp->tp_name, attr_name); +#else + "'%.50s' object has no attribute '%.400s'", + tp->tp_name, PyString_AS_STRING(attr_name)); +#endif + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name) { + PyObject *descr; + PyTypeObject *tp = Py_TYPE(obj); + if (unlikely(!PyString_Check(attr_name))) { + return PyObject_GenericGetAttr(obj, attr_name); + } + assert(!tp->tp_dictoffset); + descr = _PyType_Lookup(tp, attr_name); + if (unlikely(!descr)) { + return __Pyx_RaiseGenericGetAttributeError(tp, attr_name); + } + Py_INCREF(descr); + #if PY_MAJOR_VERSION < 3 + if (likely(PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_HAVE_CLASS))) + #endif + { + descrgetfunc f = Py_TYPE(descr)->tp_descr_get; + if (unlikely(f)) { + PyObject *res = f(descr, obj, (PyObject *)tp); + Py_DECREF(descr); + return res; + } + } + return descr; +} +#endif + /* TypeImport */ #ifndef __PYX_HAVE_RT_ImportType #define __PYX_HAVE_RT_ImportType @@ -7593,6 +8777,37 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value) } } +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { + const long neg_one = (long) ((long) 0 - (long) 1), const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); + } +} + /* CIntToPy */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { const int neg_one = (int) ((int) 0 - (int) 1), const_zero = (int) 0; @@ -7763,7 +8978,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { theta = 0; } else { r = -a.real; - theta = atan2f(0, -1); + theta = atan2f(0.0, -1.0); } } else { r = __Pyx_c_abs_float(a); @@ -7918,7 +9133,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { theta = 0; } else { r = -a.real; - theta = atan2(0, -1); + theta = atan2(0.0, -1.0); } } else { r = __Pyx_c_abs_double(a); @@ -8532,37 +9747,6 @@ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { return (int) -1; } -/* CIntToPy */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { - const long neg_one = (long) ((long) 0 - (long) 1), const_zero = (long) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(long) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(long) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(long) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(long), - little, !is_unsigned); - } -} - /* CIntFromPy */ static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { const long neg_one = (long) ((long) 0 - (long) 1), const_zero = (long) 0; diff --git a/gensim/models/_utils_any2vec.pyx b/gensim/models/_utils_any2vec.pyx index 674cfe8146..1cafacb6fc 100644 --- a/gensim/models/_utils_any2vec.pyx +++ b/gensim/models/_utils_any2vec.pyx @@ -101,6 +101,17 @@ _MB_MASK = 0xC0 _MB_START = 0x80 +cdef _is_utf8_continue_py3(b): + return b & _MB_MASK == _MB_START + + +cdef _is_utf8_continue_py2(b): + return ord(b) & _MB_MASK == _MB_START + + +_is_utf8_continue = _is_utf8_continue_py2 if PY2 else _is_utf8_continue_py3 + + cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): """Computes ngrams for a word. @@ -131,13 +142,13 @@ cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): ngrams = [] for i in range(num_bytes): - if utf8_word[i] & _MB_MASK == _MB_START: + if _is_utf8_continue(utf8_word[i]): continue j, n = i, 1 while j < num_bytes and n <= max_n: j += 1 - while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: + while j < num_bytes and _is_utf8_continue(utf8_word[j]): j += 1 if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): ngram = bytes(utf8_word[i:j]) diff --git a/gensim/models/utils_any2vec.py b/gensim/models/utils_any2vec.py index 0e77840137..9108e457a4 100644 --- a/gensim/models/utils_any2vec.py +++ b/gensim/models/utils_any2vec.py @@ -47,8 +47,8 @@ def _byte_to_int_py3(b): return b - - + + def _byte_to_int_py2(b): return ord(b) @@ -56,6 +56,11 @@ def _byte_to_int_py2(b): _byte_to_int = _byte_to_int_py2 if PY2 else _byte_to_int_py3 +def _is_utf8_continue(b): + return _byte_to_int(b) & _MB_MASK == _MB_START + + + # # Define this here so we can unittest this function directly. # Only use this function if the faster C version fails to import. @@ -167,13 +172,13 @@ def _compute_ngrams_bytes_py(word, min_n, max_n): ngrams = [] for i in range(num_bytes): - if utf8_word[i] & _MB_MASK == _MB_START: + if _is_utf8_continue(utf8_word[i]): continue j, n = i, 1 while j < num_bytes and n <= max_n: j += 1 - while j < num_bytes and (utf8_word[j] & _MB_MASK) == _MB_START: + while j < num_bytes and _is_utf8_continue(utf8_word[j]): j += 1 if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): ngram = bytes(utf8_word[i:j]) From 6617b808b2041fe16134ce53877d03e0adceb59f Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Wed, 23 Jan 2019 14:17:34 +1100 Subject: [PATCH 24/32] flake8 --- gensim/models/utils_any2vec.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gensim/models/utils_any2vec.py b/gensim/models/utils_any2vec.py index 9108e457a4..88242c31ea 100644 --- a/gensim/models/utils_any2vec.py +++ b/gensim/models/utils_any2vec.py @@ -47,8 +47,8 @@ def _byte_to_int_py3(b): return b - - + + def _byte_to_int_py2(b): return ord(b) @@ -60,7 +60,6 @@ def _is_utf8_continue(b): return _byte_to_int(b) & _MB_MASK == _MB_START - # # Define this here so we can unittest this function directly. # Only use this function if the faster C version fails to import. From da025d33b6797b524c092e3e65f51895052f4003 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Wed, 23 Jan 2019 15:12:41 +1100 Subject: [PATCH 25/32] review response: get rid of set() --- gensim/test/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gensim/test/test_utils.py b/gensim/test/test_utils.py index 3153195092..a71b828887 100644 --- a/gensim/test/test_utils.py +++ b/gensim/test/test_utils.py @@ -463,7 +463,7 @@ def test_bytes_py(self): # The text and byte implementations yield ngrams in different # order, so the test ignores ngram order. # - self.assertEqual(sorted(set(expected_text)), sorted(set(actual_text))) + self.assertEqual(sorted(expected_text), sorted(actual_text)) @unittest.skipIf(DISABLE_CYTHON_TESTS, 'Cython functions are not properly compiled') def test_bytes_cy(self): @@ -474,7 +474,7 @@ def test_bytes_cy(self): expected_text = self.expected_text[word] actual_text = [n.decode('utf-8') for n in actual] - self.assertEqual(sorted(set(expected_text)), sorted(set(actual_text))) + self.assertEqual(sorted(expected_text), sorted(actual_text)) if __name__ == '__main__': From 402db7d285f5230da65c12ac6e7c41fe485801a1 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Wed, 23 Jan 2019 15:14:21 +1100 Subject: [PATCH 26/32] review response: remove excess bytes() call --- gensim/models/_utils_any2vec.c | 449 +++++++++++++++---------------- gensim/models/_utils_any2vec.pyx | 2 +- 2 files changed, 224 insertions(+), 227 deletions(-) diff --git a/gensim/models/_utils_any2vec.c b/gensim/models/_utils_any2vec.c index ad8cdfe979..7e7ca73364 100644 --- a/gensim/models/_utils_any2vec.c +++ b/gensim/models/_utils_any2vec.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.29.3 */ +/* Generated by Cython 0.29.2 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -7,8 +7,8 @@ #elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03030000) #error Cython requires Python 2.6+ or Python 3.3+. #else -#define CYTHON_ABI "0_29_3" -#define CYTHON_HEX_VERSION 0x001D03F0 +#define CYTHON_ABI "0_29_2" +#define CYTHON_HEX_VERSION 0x001D02F0 #define CYTHON_FUTURE_DIVISION 0 #include #ifndef offsetof @@ -398,7 +398,7 @@ typedef int Py_tss_t; static CYTHON_INLINE int PyThread_tss_create(Py_tss_t *key) { *key = PyThread_create_key(); - return 0; + return 0; // PyThread_create_key reports success always } static CYTHON_INLINE Py_tss_t * PyThread_tss_alloc(void) { Py_tss_t *key = (Py_tss_t *)PyObject_Malloc(sizeof(Py_tss_t)); @@ -421,7 +421,7 @@ static CYTHON_INLINE int PyThread_tss_set(Py_tss_t *key, void *value) { static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { return PyThread_get_key_value(*key); } -#endif +#endif // TSS (Thread Specific Storage) API #if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) #define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) #else @@ -848,7 +848,7 @@ static const char *__pyx_f[] = { "type.pxd", }; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":776 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":776 * # in Cython to enable them only on the right systems. * * ctypedef npy_int8 int8_t # <<<<<<<<<<<<<< @@ -857,7 +857,7 @@ static const char *__pyx_f[] = { */ typedef npy_int8 __pyx_t_5numpy_int8_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":777 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":777 * * ctypedef npy_int8 int8_t * ctypedef npy_int16 int16_t # <<<<<<<<<<<<<< @@ -866,7 +866,7 @@ typedef npy_int8 __pyx_t_5numpy_int8_t; */ typedef npy_int16 __pyx_t_5numpy_int16_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":778 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":778 * ctypedef npy_int8 int8_t * ctypedef npy_int16 int16_t * ctypedef npy_int32 int32_t # <<<<<<<<<<<<<< @@ -875,7 +875,7 @@ typedef npy_int16 __pyx_t_5numpy_int16_t; */ typedef npy_int32 __pyx_t_5numpy_int32_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":779 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":779 * ctypedef npy_int16 int16_t * ctypedef npy_int32 int32_t * ctypedef npy_int64 int64_t # <<<<<<<<<<<<<< @@ -884,7 +884,7 @@ typedef npy_int32 __pyx_t_5numpy_int32_t; */ typedef npy_int64 __pyx_t_5numpy_int64_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":783 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":783 * #ctypedef npy_int128 int128_t * * ctypedef npy_uint8 uint8_t # <<<<<<<<<<<<<< @@ -893,7 +893,7 @@ typedef npy_int64 __pyx_t_5numpy_int64_t; */ typedef npy_uint8 __pyx_t_5numpy_uint8_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":784 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":784 * * ctypedef npy_uint8 uint8_t * ctypedef npy_uint16 uint16_t # <<<<<<<<<<<<<< @@ -902,7 +902,7 @@ typedef npy_uint8 __pyx_t_5numpy_uint8_t; */ typedef npy_uint16 __pyx_t_5numpy_uint16_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":785 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":785 * ctypedef npy_uint8 uint8_t * ctypedef npy_uint16 uint16_t * ctypedef npy_uint32 uint32_t # <<<<<<<<<<<<<< @@ -911,7 +911,7 @@ typedef npy_uint16 __pyx_t_5numpy_uint16_t; */ typedef npy_uint32 __pyx_t_5numpy_uint32_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":786 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":786 * ctypedef npy_uint16 uint16_t * ctypedef npy_uint32 uint32_t * ctypedef npy_uint64 uint64_t # <<<<<<<<<<<<<< @@ -920,7 +920,7 @@ typedef npy_uint32 __pyx_t_5numpy_uint32_t; */ typedef npy_uint64 __pyx_t_5numpy_uint64_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":790 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":790 * #ctypedef npy_uint128 uint128_t * * ctypedef npy_float32 float32_t # <<<<<<<<<<<<<< @@ -929,7 +929,7 @@ typedef npy_uint64 __pyx_t_5numpy_uint64_t; */ typedef npy_float32 __pyx_t_5numpy_float32_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":791 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":791 * * ctypedef npy_float32 float32_t * ctypedef npy_float64 float64_t # <<<<<<<<<<<<<< @@ -938,7 +938,7 @@ typedef npy_float32 __pyx_t_5numpy_float32_t; */ typedef npy_float64 __pyx_t_5numpy_float64_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":800 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":800 * # The int types are mapped a bit surprising -- * # numpy.int corresponds to 'l' and numpy.long to 'q' * ctypedef npy_long int_t # <<<<<<<<<<<<<< @@ -947,7 +947,7 @@ typedef npy_float64 __pyx_t_5numpy_float64_t; */ typedef npy_long __pyx_t_5numpy_int_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":801 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":801 * # numpy.int corresponds to 'l' and numpy.long to 'q' * ctypedef npy_long int_t * ctypedef npy_longlong long_t # <<<<<<<<<<<<<< @@ -956,7 +956,7 @@ typedef npy_long __pyx_t_5numpy_int_t; */ typedef npy_longlong __pyx_t_5numpy_long_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":802 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":802 * ctypedef npy_long int_t * ctypedef npy_longlong long_t * ctypedef npy_longlong longlong_t # <<<<<<<<<<<<<< @@ -965,7 +965,7 @@ typedef npy_longlong __pyx_t_5numpy_long_t; */ typedef npy_longlong __pyx_t_5numpy_longlong_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":804 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":804 * ctypedef npy_longlong longlong_t * * ctypedef npy_ulong uint_t # <<<<<<<<<<<<<< @@ -974,7 +974,7 @@ typedef npy_longlong __pyx_t_5numpy_longlong_t; */ typedef npy_ulong __pyx_t_5numpy_uint_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":805 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":805 * * ctypedef npy_ulong uint_t * ctypedef npy_ulonglong ulong_t # <<<<<<<<<<<<<< @@ -983,7 +983,7 @@ typedef npy_ulong __pyx_t_5numpy_uint_t; */ typedef npy_ulonglong __pyx_t_5numpy_ulong_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":806 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":806 * ctypedef npy_ulong uint_t * ctypedef npy_ulonglong ulong_t * ctypedef npy_ulonglong ulonglong_t # <<<<<<<<<<<<<< @@ -992,7 +992,7 @@ typedef npy_ulonglong __pyx_t_5numpy_ulong_t; */ typedef npy_ulonglong __pyx_t_5numpy_ulonglong_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":808 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":808 * ctypedef npy_ulonglong ulonglong_t * * ctypedef npy_intp intp_t # <<<<<<<<<<<<<< @@ -1001,7 +1001,7 @@ typedef npy_ulonglong __pyx_t_5numpy_ulonglong_t; */ typedef npy_intp __pyx_t_5numpy_intp_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":809 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":809 * * ctypedef npy_intp intp_t * ctypedef npy_uintp uintp_t # <<<<<<<<<<<<<< @@ -1010,7 +1010,7 @@ typedef npy_intp __pyx_t_5numpy_intp_t; */ typedef npy_uintp __pyx_t_5numpy_uintp_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":811 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":811 * ctypedef npy_uintp uintp_t * * ctypedef npy_double float_t # <<<<<<<<<<<<<< @@ -1019,7 +1019,7 @@ typedef npy_uintp __pyx_t_5numpy_uintp_t; */ typedef npy_double __pyx_t_5numpy_float_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":812 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":812 * * ctypedef npy_double float_t * ctypedef npy_double double_t # <<<<<<<<<<<<<< @@ -1028,7 +1028,7 @@ typedef npy_double __pyx_t_5numpy_float_t; */ typedef npy_double __pyx_t_5numpy_double_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":813 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":813 * ctypedef npy_double float_t * ctypedef npy_double double_t * ctypedef npy_longdouble longdouble_t # <<<<<<<<<<<<<< @@ -1064,7 +1064,7 @@ static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(do /*--- Type declarations ---*/ struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":815 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":815 * ctypedef npy_longdouble longdouble_t * * ctypedef npy_cfloat cfloat_t # <<<<<<<<<<<<<< @@ -1073,7 +1073,7 @@ struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py; */ typedef npy_cfloat __pyx_t_5numpy_cfloat_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":816 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":816 * * ctypedef npy_cfloat cfloat_t * ctypedef npy_cdouble cdouble_t # <<<<<<<<<<<<<< @@ -1082,7 +1082,7 @@ typedef npy_cfloat __pyx_t_5numpy_cfloat_t; */ typedef npy_cdouble __pyx_t_5numpy_cdouble_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":817 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":817 * ctypedef npy_cfloat cfloat_t * ctypedef npy_cdouble cdouble_t * ctypedef npy_clongdouble clongdouble_t # <<<<<<<<<<<<<< @@ -1091,7 +1091,7 @@ typedef npy_cdouble __pyx_t_5numpy_cdouble_t; */ typedef npy_clongdouble __pyx_t_5numpy_clongdouble_t; -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":819 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":819 * ctypedef npy_clongdouble clongdouble_t * * ctypedef npy_cdouble complex_t # <<<<<<<<<<<<<< @@ -2934,7 +2934,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< * ngram = bytes(utf8_word[i:j]) - * ngrams.append(bytes(ngram)) + * ngrams.append(ngram) */ __pyx_t_12 = ((__pyx_v_n >= __pyx_v_min_n) != 0); if (__pyx_t_12) { @@ -2966,7 +2966,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): * ngram = bytes(utf8_word[i:j]) # <<<<<<<<<<<<<< - * ngrams.append(bytes(ngram)) + * ngrams.append(ngram) * n += 1 */ if (unlikely(__pyx_v_utf8_word == Py_None)) { @@ -2984,27 +2984,24 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P /* "gensim/models/_utils_any2vec.pyx":155 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): * ngram = bytes(utf8_word[i:j]) - * ngrams.append(bytes(ngram)) # <<<<<<<<<<<<<< + * ngrams.append(ngram) # <<<<<<<<<<<<<< * n += 1 * return ngrams */ - __pyx_t_3 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_v_ngram); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 155, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_t_3); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 155, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_v_ngram); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 155, __pyx_L1_error) /* "gensim/models/_utils_any2vec.pyx":153 * while j < num_bytes and _is_utf8_continue(utf8_word[j]): * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< * ngram = bytes(utf8_word[i:j]) - * ngrams.append(bytes(ngram)) + * ngrams.append(ngram) */ } /* "gensim/models/_utils_any2vec.pyx":156 * ngram = bytes(utf8_word[i:j]) - * ngrams.append(bytes(ngram)) + * ngrams.append(ngram) * n += 1 # <<<<<<<<<<<<<< * return ngrams */ @@ -3014,7 +3011,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P } /* "gensim/models/_utils_any2vec.pyx":157 - * ngrams.append(bytes(ngram)) + * ngrams.append(ngram) * n += 1 * return ngrams # <<<<<<<<<<<<<< */ @@ -3144,7 +3141,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes return __pyx_r; } -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 * # experimental exception made for __getbuffer__ and __releasebuffer__ * # -- the details of this may change. * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< @@ -3193,7 +3190,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); __Pyx_GIVEREF(__pyx_v_info->obj); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":265 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":265 * * cdef int i, ndim * cdef int endian_detector = 1 # <<<<<<<<<<<<<< @@ -3202,7 +3199,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_endian_detector = 1; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":266 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":266 * cdef int i, ndim * cdef int endian_detector = 1 * cdef bint little_endian = ((&endian_detector)[0] != 0) # <<<<<<<<<<<<<< @@ -3211,7 +3208,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_little_endian = ((((char *)(&__pyx_v_endian_detector))[0]) != 0); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":268 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":268 * cdef bint little_endian = ((&endian_detector)[0] != 0) * * ndim = PyArray_NDIM(self) # <<<<<<<<<<<<<< @@ -3220,7 +3217,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_ndim = PyArray_NDIM(__pyx_v_self); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 * ndim = PyArray_NDIM(self) * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -3234,7 +3231,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P goto __pyx_L4_bool_binop_done; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":271 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":271 * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): # <<<<<<<<<<<<<< @@ -3245,7 +3242,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = __pyx_t_2; __pyx_L4_bool_binop_done:; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 * ndim = PyArray_NDIM(self) * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -3254,7 +3251,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ if (unlikely(__pyx_t_1)) { - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): * raise ValueError(u"ndarray is not C contiguous") # <<<<<<<<<<<<<< @@ -3267,7 +3264,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 272, __pyx_L1_error) - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 * ndim = PyArray_NDIM(self) * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -3276,7 +3273,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 * raise ValueError(u"ndarray is not C contiguous") * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -3290,7 +3287,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P goto __pyx_L7_bool_binop_done; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":275 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":275 * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): # <<<<<<<<<<<<<< @@ -3301,7 +3298,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = __pyx_t_2; __pyx_L7_bool_binop_done:; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 * raise ValueError(u"ndarray is not C contiguous") * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -3310,7 +3307,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ if (unlikely(__pyx_t_1)) { - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): * raise ValueError(u"ndarray is not Fortran contiguous") # <<<<<<<<<<<<<< @@ -3323,7 +3320,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 276, __pyx_L1_error) - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 * raise ValueError(u"ndarray is not C contiguous") * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< @@ -3332,7 +3329,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":278 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":278 * raise ValueError(u"ndarray is not Fortran contiguous") * * info.buf = PyArray_DATA(self) # <<<<<<<<<<<<<< @@ -3341,7 +3338,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->buf = PyArray_DATA(__pyx_v_self); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":279 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":279 * * info.buf = PyArray_DATA(self) * info.ndim = ndim # <<<<<<<<<<<<<< @@ -3350,7 +3347,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->ndim = __pyx_v_ndim; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 * info.buf = PyArray_DATA(self) * info.ndim = ndim * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< @@ -3360,7 +3357,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = (((sizeof(npy_intp)) != (sizeof(Py_ssize_t))) != 0); if (__pyx_t_1) { - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":283 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":283 * # Allocate new buffer for strides and shape info. * # This is allocated as one block, strides first. * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) # <<<<<<<<<<<<<< @@ -3369,7 +3366,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->strides = ((Py_ssize_t *)PyObject_Malloc((((sizeof(Py_ssize_t)) * 2) * ((size_t)__pyx_v_ndim)))); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":284 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":284 * # This is allocated as one block, strides first. * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) * info.shape = info.strides + ndim # <<<<<<<<<<<<<< @@ -3378,7 +3375,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->shape = (__pyx_v_info->strides + __pyx_v_ndim); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":285 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":285 * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) * info.shape = info.strides + ndim * for i in range(ndim): # <<<<<<<<<<<<<< @@ -3390,7 +3387,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { __pyx_v_i = __pyx_t_6; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":286 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":286 * info.shape = info.strides + ndim * for i in range(ndim): * info.strides[i] = PyArray_STRIDES(self)[i] # <<<<<<<<<<<<<< @@ -3399,7 +3396,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ (__pyx_v_info->strides[__pyx_v_i]) = (PyArray_STRIDES(__pyx_v_self)[__pyx_v_i]); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":287 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":287 * for i in range(ndim): * info.strides[i] = PyArray_STRIDES(self)[i] * info.shape[i] = PyArray_DIMS(self)[i] # <<<<<<<<<<<<<< @@ -3409,7 +3406,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P (__pyx_v_info->shape[__pyx_v_i]) = (PyArray_DIMS(__pyx_v_self)[__pyx_v_i]); } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 * info.buf = PyArray_DATA(self) * info.ndim = ndim * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< @@ -3419,7 +3416,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P goto __pyx_L9; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":289 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":289 * info.shape[i] = PyArray_DIMS(self)[i] * else: * info.strides = PyArray_STRIDES(self) # <<<<<<<<<<<<<< @@ -3429,7 +3426,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P /*else*/ { __pyx_v_info->strides = ((Py_ssize_t *)PyArray_STRIDES(__pyx_v_self)); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":290 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":290 * else: * info.strides = PyArray_STRIDES(self) * info.shape = PyArray_DIMS(self) # <<<<<<<<<<<<<< @@ -3440,7 +3437,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P } __pyx_L9:; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":291 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":291 * info.strides = PyArray_STRIDES(self) * info.shape = PyArray_DIMS(self) * info.suboffsets = NULL # <<<<<<<<<<<<<< @@ -3449,7 +3446,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->suboffsets = NULL; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":292 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":292 * info.shape = PyArray_DIMS(self) * info.suboffsets = NULL * info.itemsize = PyArray_ITEMSIZE(self) # <<<<<<<<<<<<<< @@ -3458,7 +3455,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->itemsize = PyArray_ITEMSIZE(__pyx_v_self); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":293 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":293 * info.suboffsets = NULL * info.itemsize = PyArray_ITEMSIZE(self) * info.readonly = not PyArray_ISWRITEABLE(self) # <<<<<<<<<<<<<< @@ -3467,7 +3464,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->readonly = (!(PyArray_ISWRITEABLE(__pyx_v_self) != 0)); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":296 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":296 * * cdef int t * cdef char* f = NULL # <<<<<<<<<<<<<< @@ -3476,7 +3473,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_f = NULL; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":297 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":297 * cdef int t * cdef char* f = NULL * cdef dtype descr = PyArray_DESCR(self) # <<<<<<<<<<<<<< @@ -3489,7 +3486,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_v_descr = ((PyArray_Descr *)__pyx_t_3); __pyx_t_3 = 0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":300 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":300 * cdef int offset * * info.obj = self # <<<<<<<<<<<<<< @@ -3502,7 +3499,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = ((PyObject *)__pyx_v_self); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 * info.obj = self * * if not PyDataType_HASFIELDS(descr): # <<<<<<<<<<<<<< @@ -3512,7 +3509,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = ((!(PyDataType_HASFIELDS(__pyx_v_descr) != 0)) != 0); if (__pyx_t_1) { - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":303 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":303 * * if not PyDataType_HASFIELDS(descr): * t = descr.type_num # <<<<<<<<<<<<<< @@ -3522,7 +3519,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_4 = __pyx_v_descr->type_num; __pyx_v_t = __pyx_t_4; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 * if not PyDataType_HASFIELDS(descr): * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -3542,7 +3539,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P } __pyx_L15_next_or:; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":305 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":305 * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or * (descr.byteorder == c'<' and not little_endian)): # <<<<<<<<<<<<<< @@ -3559,7 +3556,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_1 = __pyx_t_2; __pyx_L14_bool_binop_done:; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 * if not PyDataType_HASFIELDS(descr): * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -3568,7 +3565,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ if (unlikely(__pyx_t_1)) { - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 * if ((descr.byteorder == c'>' and little_endian) or * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< @@ -3581,7 +3578,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 306, __pyx_L1_error) - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 * if not PyDataType_HASFIELDS(descr): * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -3590,7 +3587,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":307 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":307 * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") * if t == NPY_BYTE: f = "b" # <<<<<<<<<<<<<< @@ -3603,7 +3600,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_UBYTE: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":308 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":308 * raise ValueError(u"Non-native byte order not supported") * if t == NPY_BYTE: f = "b" * elif t == NPY_UBYTE: f = "B" # <<<<<<<<<<<<<< @@ -3614,7 +3611,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_SHORT: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":309 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":309 * if t == NPY_BYTE: f = "b" * elif t == NPY_UBYTE: f = "B" * elif t == NPY_SHORT: f = "h" # <<<<<<<<<<<<<< @@ -3625,7 +3622,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_USHORT: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":310 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":310 * elif t == NPY_UBYTE: f = "B" * elif t == NPY_SHORT: f = "h" * elif t == NPY_USHORT: f = "H" # <<<<<<<<<<<<<< @@ -3636,7 +3633,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_INT: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":311 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":311 * elif t == NPY_SHORT: f = "h" * elif t == NPY_USHORT: f = "H" * elif t == NPY_INT: f = "i" # <<<<<<<<<<<<<< @@ -3647,7 +3644,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_UINT: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":312 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":312 * elif t == NPY_USHORT: f = "H" * elif t == NPY_INT: f = "i" * elif t == NPY_UINT: f = "I" # <<<<<<<<<<<<<< @@ -3658,7 +3655,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_LONG: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":313 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":313 * elif t == NPY_INT: f = "i" * elif t == NPY_UINT: f = "I" * elif t == NPY_LONG: f = "l" # <<<<<<<<<<<<<< @@ -3669,7 +3666,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_ULONG: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":314 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":314 * elif t == NPY_UINT: f = "I" * elif t == NPY_LONG: f = "l" * elif t == NPY_ULONG: f = "L" # <<<<<<<<<<<<<< @@ -3680,7 +3677,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_LONGLONG: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":315 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":315 * elif t == NPY_LONG: f = "l" * elif t == NPY_ULONG: f = "L" * elif t == NPY_LONGLONG: f = "q" # <<<<<<<<<<<<<< @@ -3691,7 +3688,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_ULONGLONG: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":316 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":316 * elif t == NPY_ULONG: f = "L" * elif t == NPY_LONGLONG: f = "q" * elif t == NPY_ULONGLONG: f = "Q" # <<<<<<<<<<<<<< @@ -3702,7 +3699,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_FLOAT: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":317 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":317 * elif t == NPY_LONGLONG: f = "q" * elif t == NPY_ULONGLONG: f = "Q" * elif t == NPY_FLOAT: f = "f" # <<<<<<<<<<<<<< @@ -3713,7 +3710,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_DOUBLE: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":318 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":318 * elif t == NPY_ULONGLONG: f = "Q" * elif t == NPY_FLOAT: f = "f" * elif t == NPY_DOUBLE: f = "d" # <<<<<<<<<<<<<< @@ -3724,7 +3721,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_LONGDOUBLE: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":319 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":319 * elif t == NPY_FLOAT: f = "f" * elif t == NPY_DOUBLE: f = "d" * elif t == NPY_LONGDOUBLE: f = "g" # <<<<<<<<<<<<<< @@ -3735,7 +3732,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_CFLOAT: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":320 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":320 * elif t == NPY_DOUBLE: f = "d" * elif t == NPY_LONGDOUBLE: f = "g" * elif t == NPY_CFLOAT: f = "Zf" # <<<<<<<<<<<<<< @@ -3746,7 +3743,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_CDOUBLE: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":321 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":321 * elif t == NPY_LONGDOUBLE: f = "g" * elif t == NPY_CFLOAT: f = "Zf" * elif t == NPY_CDOUBLE: f = "Zd" # <<<<<<<<<<<<<< @@ -3757,7 +3754,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_CLONGDOUBLE: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":322 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":322 * elif t == NPY_CFLOAT: f = "Zf" * elif t == NPY_CDOUBLE: f = "Zd" * elif t == NPY_CLONGDOUBLE: f = "Zg" # <<<<<<<<<<<<<< @@ -3768,7 +3765,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; case NPY_OBJECT: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":323 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":323 * elif t == NPY_CDOUBLE: f = "Zd" * elif t == NPY_CLONGDOUBLE: f = "Zg" * elif t == NPY_OBJECT: f = "O" # <<<<<<<<<<<<<< @@ -3779,7 +3776,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; default: - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":325 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":325 * elif t == NPY_OBJECT: f = "O" * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) # <<<<<<<<<<<<<< @@ -3800,7 +3797,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P break; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":326 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":326 * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) * info.format = f # <<<<<<<<<<<<<< @@ -3809,7 +3806,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_info->format = __pyx_v_f; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":327 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":327 * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) * info.format = f * return # <<<<<<<<<<<<<< @@ -3819,7 +3816,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_r = 0; goto __pyx_L0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 * info.obj = self * * if not PyDataType_HASFIELDS(descr): # <<<<<<<<<<<<<< @@ -3828,7 +3825,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":329 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":329 * return * else: * info.format = PyObject_Malloc(_buffer_format_string_len) # <<<<<<<<<<<<<< @@ -3838,7 +3835,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P /*else*/ { __pyx_v_info->format = ((char *)PyObject_Malloc(0xFF)); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":330 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":330 * else: * info.format = PyObject_Malloc(_buffer_format_string_len) * info.format[0] = c'^' # Native data types, manual alignment # <<<<<<<<<<<<<< @@ -3847,7 +3844,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ (__pyx_v_info->format[0]) = '^'; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":331 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":331 * info.format = PyObject_Malloc(_buffer_format_string_len) * info.format[0] = c'^' # Native data types, manual alignment * offset = 0 # <<<<<<<<<<<<<< @@ -3856,7 +3853,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P */ __pyx_v_offset = 0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":332 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":332 * info.format[0] = c'^' # Native data types, manual alignment * offset = 0 * f = _util_dtypestring(descr, info.format + 1, # <<<<<<<<<<<<<< @@ -3866,7 +3863,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P __pyx_t_9 = __pyx_f_5numpy__util_dtypestring(__pyx_v_descr, (__pyx_v_info->format + 1), (__pyx_v_info->format + 0xFF), (&__pyx_v_offset)); if (unlikely(__pyx_t_9 == ((char *)NULL))) __PYX_ERR(1, 332, __pyx_L1_error) __pyx_v_f = __pyx_t_9; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":335 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":335 * info.format + _buffer_format_string_len, * &offset) * f[0] = c'\0' # Terminate format string # <<<<<<<<<<<<<< @@ -3876,7 +3873,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P (__pyx_v_f[0]) = '\x00'; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 * # experimental exception made for __getbuffer__ and __releasebuffer__ * # -- the details of this may change. * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< @@ -3908,7 +3905,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P return __pyx_r; } -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 * f[0] = c'\0' # Terminate format string * * def __releasebuffer__(ndarray self, Py_buffer* info): # <<<<<<<<<<<<<< @@ -3932,7 +3929,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s int __pyx_t_1; __Pyx_RefNannySetupContext("__releasebuffer__", 0); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 * * def __releasebuffer__(ndarray self, Py_buffer* info): * if PyArray_HASFIELDS(self): # <<<<<<<<<<<<<< @@ -3942,7 +3939,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s __pyx_t_1 = (PyArray_HASFIELDS(__pyx_v_self) != 0); if (__pyx_t_1) { - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":339 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":339 * def __releasebuffer__(ndarray self, Py_buffer* info): * if PyArray_HASFIELDS(self): * PyObject_Free(info.format) # <<<<<<<<<<<<<< @@ -3951,7 +3948,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s */ PyObject_Free(__pyx_v_info->format); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 * * def __releasebuffer__(ndarray self, Py_buffer* info): * if PyArray_HASFIELDS(self): # <<<<<<<<<<<<<< @@ -3960,7 +3957,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s */ } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 * if PyArray_HASFIELDS(self): * PyObject_Free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< @@ -3970,7 +3967,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s __pyx_t_1 = (((sizeof(npy_intp)) != (sizeof(Py_ssize_t))) != 0); if (__pyx_t_1) { - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":341 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":341 * PyObject_Free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): * PyObject_Free(info.strides) # <<<<<<<<<<<<<< @@ -3979,7 +3976,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s */ PyObject_Free(__pyx_v_info->strides); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 * if PyArray_HASFIELDS(self): * PyObject_Free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< @@ -3988,7 +3985,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s */ } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 * f[0] = c'\0' # Terminate format string * * def __releasebuffer__(ndarray self, Py_buffer* info): # <<<<<<<<<<<<<< @@ -4000,7 +3997,7 @@ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_s __Pyx_RefNannyFinishContext(); } -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 * ctypedef npy_cdouble complex_t * * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< @@ -4014,7 +4011,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew1", 0); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":822 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":822 * * cdef inline object PyArray_MultiIterNew1(a): * return PyArray_MultiIterNew(1, a) # <<<<<<<<<<<<<< @@ -4028,7 +4025,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 * ctypedef npy_cdouble complex_t * * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< @@ -4047,7 +4044,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__ return __pyx_r; } -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 * return PyArray_MultiIterNew(1, a) * * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< @@ -4061,7 +4058,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew2", 0); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":825 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":825 * * cdef inline object PyArray_MultiIterNew2(a, b): * return PyArray_MultiIterNew(2, a, b) # <<<<<<<<<<<<<< @@ -4075,7 +4072,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 * return PyArray_MultiIterNew(1, a) * * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< @@ -4094,7 +4091,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__ return __pyx_r; } -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":827 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":827 * return PyArray_MultiIterNew(2, a, b) * * cdef inline object PyArray_MultiIterNew3(a, b, c): # <<<<<<<<<<<<<< @@ -4108,7 +4105,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew3(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew3", 0); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":828 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":828 * * cdef inline object PyArray_MultiIterNew3(a, b, c): * return PyArray_MultiIterNew(3, a, b, c) # <<<<<<<<<<<<<< @@ -4122,7 +4119,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew3(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":827 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":827 * return PyArray_MultiIterNew(2, a, b) * * cdef inline object PyArray_MultiIterNew3(a, b, c): # <<<<<<<<<<<<<< @@ -4141,7 +4138,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew3(PyObject *__ return __pyx_r; } -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":830 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":830 * return PyArray_MultiIterNew(3, a, b, c) * * cdef inline object PyArray_MultiIterNew4(a, b, c, d): # <<<<<<<<<<<<<< @@ -4155,7 +4152,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew4(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew4", 0); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":831 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":831 * * cdef inline object PyArray_MultiIterNew4(a, b, c, d): * return PyArray_MultiIterNew(4, a, b, c, d) # <<<<<<<<<<<<<< @@ -4169,7 +4166,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew4(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":830 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":830 * return PyArray_MultiIterNew(3, a, b, c) * * cdef inline object PyArray_MultiIterNew4(a, b, c, d): # <<<<<<<<<<<<<< @@ -4188,7 +4185,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew4(PyObject *__ return __pyx_r; } -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":833 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":833 * return PyArray_MultiIterNew(4, a, b, c, d) * * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): # <<<<<<<<<<<<<< @@ -4202,7 +4199,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew5(PyObject *__ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew5", 0); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":834 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":834 * * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): * return PyArray_MultiIterNew(5, a, b, c, d, e) # <<<<<<<<<<<<<< @@ -4216,7 +4213,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew5(PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":833 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":833 * return PyArray_MultiIterNew(4, a, b, c, d) * * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): # <<<<<<<<<<<<<< @@ -4235,7 +4232,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew5(PyObject *__ return __pyx_r; } -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":836 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":836 * return PyArray_MultiIterNew(5, a, b, c, d, e) * * cdef inline tuple PyDataType_SHAPE(dtype d): # <<<<<<<<<<<<<< @@ -4249,7 +4246,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ int __pyx_t_1; __Pyx_RefNannySetupContext("PyDataType_SHAPE", 0); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":837 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":837 * * cdef inline tuple PyDataType_SHAPE(dtype d): * if PyDataType_HASSUBARRAY(d): # <<<<<<<<<<<<<< @@ -4259,7 +4256,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ __pyx_t_1 = (PyDataType_HASSUBARRAY(__pyx_v_d) != 0); if (__pyx_t_1) { - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":838 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":838 * cdef inline tuple PyDataType_SHAPE(dtype d): * if PyDataType_HASSUBARRAY(d): * return d.subarray.shape # <<<<<<<<<<<<<< @@ -4271,7 +4268,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ __pyx_r = ((PyObject*)__pyx_v_d->subarray->shape); goto __pyx_L0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":837 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":837 * * cdef inline tuple PyDataType_SHAPE(dtype d): * if PyDataType_HASSUBARRAY(d): # <<<<<<<<<<<<<< @@ -4280,7 +4277,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ */ } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":840 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":840 * return d.subarray.shape * else: * return () # <<<<<<<<<<<<<< @@ -4294,7 +4291,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ goto __pyx_L0; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":836 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":836 * return PyArray_MultiIterNew(5, a, b, c, d, e) * * cdef inline tuple PyDataType_SHAPE(dtype d): # <<<<<<<<<<<<<< @@ -4309,7 +4306,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__ return __pyx_r; } -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":842 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":842 * return () * * cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset) except NULL: # <<<<<<<<<<<<<< @@ -4338,7 +4335,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx char *__pyx_t_9; __Pyx_RefNannySetupContext("_util_dtypestring", 0); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":847 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":847 * * cdef dtype child * cdef int endian_detector = 1 # <<<<<<<<<<<<<< @@ -4347,7 +4344,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ __pyx_v_endian_detector = 1; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":848 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":848 * cdef dtype child * cdef int endian_detector = 1 * cdef bint little_endian = ((&endian_detector)[0] != 0) # <<<<<<<<<<<<<< @@ -4356,7 +4353,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ __pyx_v_little_endian = ((((char *)(&__pyx_v_endian_detector))[0]) != 0); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":851 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":851 * cdef tuple fields * * for childname in descr.names: # <<<<<<<<<<<<<< @@ -4379,7 +4376,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_XDECREF_SET(__pyx_v_childname, __pyx_t_3); __pyx_t_3 = 0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":852 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":852 * * for childname in descr.names: * fields = descr.fields[childname] # <<<<<<<<<<<<<< @@ -4396,7 +4393,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_XDECREF_SET(__pyx_v_fields, ((PyObject*)__pyx_t_3)); __pyx_t_3 = 0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":853 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":853 * for childname in descr.names: * fields = descr.fields[childname] * child, new_offset = fields # <<<<<<<<<<<<<< @@ -4431,7 +4428,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_XDECREF_SET(__pyx_v_new_offset, __pyx_t_4); __pyx_t_4 = 0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":855 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":855 * child, new_offset = fields * * if (end - f) - (new_offset - offset[0]) < 15: # <<<<<<<<<<<<<< @@ -4448,7 +4445,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_6 = ((((__pyx_v_end - __pyx_v_f) - ((int)__pyx_t_5)) < 15) != 0); if (unlikely(__pyx_t_6)) { - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":856 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":856 * * if (end - f) - (new_offset - offset[0]) < 15: * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") # <<<<<<<<<<<<<< @@ -4461,7 +4458,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 856, __pyx_L1_error) - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":855 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":855 * child, new_offset = fields * * if (end - f) - (new_offset - offset[0]) < 15: # <<<<<<<<<<<<<< @@ -4470,7 +4467,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * * if ((child.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -4490,7 +4487,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx } __pyx_L8_next_or:; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":859 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":859 * * if ((child.byteorder == c'>' and little_endian) or * (child.byteorder == c'<' and not little_endian)): # <<<<<<<<<<<<<< @@ -4507,7 +4504,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_6 = __pyx_t_7; __pyx_L7_bool_binop_done:; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * * if ((child.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -4516,7 +4513,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ if (unlikely(__pyx_t_6)) { - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":860 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":860 * if ((child.byteorder == c'>' and little_endian) or * (child.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< @@ -4529,7 +4526,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 860, __pyx_L1_error) - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":858 * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * * if ((child.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< @@ -4538,7 +4535,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":870 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":870 * * # Output padding bytes * while offset[0] < new_offset: # <<<<<<<<<<<<<< @@ -4554,7 +4551,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (!__pyx_t_6) break; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":871 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":871 * # Output padding bytes * while offset[0] < new_offset: * f[0] = 120 # "x"; pad byte # <<<<<<<<<<<<<< @@ -4563,7 +4560,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ (__pyx_v_f[0]) = 0x78; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":872 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":872 * while offset[0] < new_offset: * f[0] = 120 # "x"; pad byte * f += 1 # <<<<<<<<<<<<<< @@ -4572,7 +4569,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ __pyx_v_f = (__pyx_v_f + 1); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":873 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":873 * f[0] = 120 # "x"; pad byte * f += 1 * offset[0] += 1 # <<<<<<<<<<<<<< @@ -4583,7 +4580,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx (__pyx_v_offset[__pyx_t_8]) = ((__pyx_v_offset[__pyx_t_8]) + 1); } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":875 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":875 * offset[0] += 1 * * offset[0] += child.itemsize # <<<<<<<<<<<<<< @@ -4593,7 +4590,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_8 = 0; (__pyx_v_offset[__pyx_t_8]) = ((__pyx_v_offset[__pyx_t_8]) + __pyx_v_child->elsize); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":877 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":877 * offset[0] += child.itemsize * * if not PyDataType_HASFIELDS(child): # <<<<<<<<<<<<<< @@ -4603,7 +4600,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_6 = ((!(PyDataType_HASFIELDS(__pyx_v_child) != 0)) != 0); if (__pyx_t_6) { - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":878 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":878 * * if not PyDataType_HASFIELDS(child): * t = child.type_num # <<<<<<<<<<<<<< @@ -4615,7 +4612,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_XDECREF_SET(__pyx_v_t, __pyx_t_4); __pyx_t_4 = 0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":879 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":879 * if not PyDataType_HASFIELDS(child): * t = child.type_num * if end - f < 5: # <<<<<<<<<<<<<< @@ -4625,7 +4622,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_t_6 = (((__pyx_v_end - __pyx_v_f) < 5) != 0); if (unlikely(__pyx_t_6)) { - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":880 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":880 * t = child.type_num * if end - f < 5: * raise RuntimeError(u"Format string allocated too short.") # <<<<<<<<<<<<<< @@ -4638,7 +4635,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __PYX_ERR(1, 880, __pyx_L1_error) - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":879 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":879 * if not PyDataType_HASFIELDS(child): * t = child.type_num * if end - f < 5: # <<<<<<<<<<<<<< @@ -4647,7 +4644,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":883 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":883 * * # Until ticket #99 is fixed, use integers to avoid warnings * if t == NPY_BYTE: f[0] = 98 #"b" # <<<<<<<<<<<<<< @@ -4665,7 +4662,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":884 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":884 * # Until ticket #99 is fixed, use integers to avoid warnings * if t == NPY_BYTE: f[0] = 98 #"b" * elif t == NPY_UBYTE: f[0] = 66 #"B" # <<<<<<<<<<<<<< @@ -4683,7 +4680,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":885 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":885 * if t == NPY_BYTE: f[0] = 98 #"b" * elif t == NPY_UBYTE: f[0] = 66 #"B" * elif t == NPY_SHORT: f[0] = 104 #"h" # <<<<<<<<<<<<<< @@ -4701,7 +4698,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":886 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":886 * elif t == NPY_UBYTE: f[0] = 66 #"B" * elif t == NPY_SHORT: f[0] = 104 #"h" * elif t == NPY_USHORT: f[0] = 72 #"H" # <<<<<<<<<<<<<< @@ -4719,7 +4716,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":887 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":887 * elif t == NPY_SHORT: f[0] = 104 #"h" * elif t == NPY_USHORT: f[0] = 72 #"H" * elif t == NPY_INT: f[0] = 105 #"i" # <<<<<<<<<<<<<< @@ -4737,7 +4734,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":888 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":888 * elif t == NPY_USHORT: f[0] = 72 #"H" * elif t == NPY_INT: f[0] = 105 #"i" * elif t == NPY_UINT: f[0] = 73 #"I" # <<<<<<<<<<<<<< @@ -4755,7 +4752,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":889 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":889 * elif t == NPY_INT: f[0] = 105 #"i" * elif t == NPY_UINT: f[0] = 73 #"I" * elif t == NPY_LONG: f[0] = 108 #"l" # <<<<<<<<<<<<<< @@ -4773,7 +4770,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":890 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":890 * elif t == NPY_UINT: f[0] = 73 #"I" * elif t == NPY_LONG: f[0] = 108 #"l" * elif t == NPY_ULONG: f[0] = 76 #"L" # <<<<<<<<<<<<<< @@ -4791,7 +4788,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":891 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":891 * elif t == NPY_LONG: f[0] = 108 #"l" * elif t == NPY_ULONG: f[0] = 76 #"L" * elif t == NPY_LONGLONG: f[0] = 113 #"q" # <<<<<<<<<<<<<< @@ -4809,7 +4806,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":892 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":892 * elif t == NPY_ULONG: f[0] = 76 #"L" * elif t == NPY_LONGLONG: f[0] = 113 #"q" * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" # <<<<<<<<<<<<<< @@ -4827,7 +4824,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":893 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":893 * elif t == NPY_LONGLONG: f[0] = 113 #"q" * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" * elif t == NPY_FLOAT: f[0] = 102 #"f" # <<<<<<<<<<<<<< @@ -4845,7 +4842,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":894 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":894 * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" * elif t == NPY_FLOAT: f[0] = 102 #"f" * elif t == NPY_DOUBLE: f[0] = 100 #"d" # <<<<<<<<<<<<<< @@ -4863,7 +4860,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":895 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":895 * elif t == NPY_FLOAT: f[0] = 102 #"f" * elif t == NPY_DOUBLE: f[0] = 100 #"d" * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" # <<<<<<<<<<<<<< @@ -4881,7 +4878,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":896 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":896 * elif t == NPY_DOUBLE: f[0] = 100 #"d" * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf # <<<<<<<<<<<<<< @@ -4901,7 +4898,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":897 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":897 * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd # <<<<<<<<<<<<<< @@ -4921,7 +4918,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":898 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":898 * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd * elif t == NPY_CLONGDOUBLE: f[0] = 90; f[1] = 103; f += 1 # Zg # <<<<<<<<<<<<<< @@ -4941,7 +4938,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":899 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":899 * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd * elif t == NPY_CLONGDOUBLE: f[0] = 90; f[1] = 103; f += 1 # Zg * elif t == NPY_OBJECT: f[0] = 79 #"O" # <<<<<<<<<<<<<< @@ -4959,7 +4956,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L15; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":901 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":901 * elif t == NPY_OBJECT: f[0] = 79 #"O" * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) # <<<<<<<<<<<<<< @@ -4978,7 +4975,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx } __pyx_L15:; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":902 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":902 * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) * f += 1 # <<<<<<<<<<<<<< @@ -4987,7 +4984,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx */ __pyx_v_f = (__pyx_v_f + 1); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":877 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":877 * offset[0] += child.itemsize * * if not PyDataType_HASFIELDS(child): # <<<<<<<<<<<<<< @@ -4997,7 +4994,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx goto __pyx_L13; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":906 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":906 * # Cython ignores struct boundary information ("T{...}"), * # so don't output it * f = _util_dtypestring(child, f, end, offset) # <<<<<<<<<<<<<< @@ -5010,7 +5007,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx } __pyx_L13:; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":851 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":851 * cdef tuple fields * * for childname in descr.names: # <<<<<<<<<<<<<< @@ -5020,7 +5017,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":907 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":907 * # so don't output it * f = _util_dtypestring(child, f, end, offset) * return f # <<<<<<<<<<<<<< @@ -5030,7 +5027,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx __pyx_r = __pyx_v_f; goto __pyx_L0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":842 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":842 * return () * * cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset) except NULL: # <<<<<<<<<<<<<< @@ -5055,7 +5052,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx return __pyx_r; } -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1022 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1022 * int _import_umath() except -1 * * cdef inline void set_array_base(ndarray arr, object base): # <<<<<<<<<<<<<< @@ -5067,7 +5064,7 @@ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_a __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("set_array_base", 0); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1023 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1023 * * cdef inline void set_array_base(ndarray arr, object base): * Py_INCREF(base) # important to do this before stealing the reference below! # <<<<<<<<<<<<<< @@ -5076,7 +5073,7 @@ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_a */ Py_INCREF(__pyx_v_base); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1024 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1024 * cdef inline void set_array_base(ndarray arr, object base): * Py_INCREF(base) # important to do this before stealing the reference below! * PyArray_SetBaseObject(arr, base) # <<<<<<<<<<<<<< @@ -5085,7 +5082,7 @@ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_a */ (void)(PyArray_SetBaseObject(__pyx_v_arr, __pyx_v_base)); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1022 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1022 * int _import_umath() except -1 * * cdef inline void set_array_base(ndarray arr, object base): # <<<<<<<<<<<<<< @@ -5097,7 +5094,7 @@ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_a __Pyx_RefNannyFinishContext(); } -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1026 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1026 * PyArray_SetBaseObject(arr, base) * * cdef inline object get_array_base(ndarray arr): # <<<<<<<<<<<<<< @@ -5112,7 +5109,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py int __pyx_t_1; __Pyx_RefNannySetupContext("get_array_base", 0); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1027 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1027 * * cdef inline object get_array_base(ndarray arr): * base = PyArray_BASE(arr) # <<<<<<<<<<<<<< @@ -5121,7 +5118,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py */ __pyx_v_base = PyArray_BASE(__pyx_v_arr); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1028 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1028 * cdef inline object get_array_base(ndarray arr): * base = PyArray_BASE(arr) * if base is NULL: # <<<<<<<<<<<<<< @@ -5131,7 +5128,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py __pyx_t_1 = ((__pyx_v_base == NULL) != 0); if (__pyx_t_1) { - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1029 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1029 * base = PyArray_BASE(arr) * if base is NULL: * return None # <<<<<<<<<<<<<< @@ -5142,7 +5139,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1028 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1028 * cdef inline object get_array_base(ndarray arr): * base = PyArray_BASE(arr) * if base is NULL: # <<<<<<<<<<<<<< @@ -5151,7 +5148,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py */ } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1030 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1030 * if base is NULL: * return None * return base # <<<<<<<<<<<<<< @@ -5163,7 +5160,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py __pyx_r = ((PyObject *)__pyx_v_base); goto __pyx_L0; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1026 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1026 * PyArray_SetBaseObject(arr, base) * * cdef inline object get_array_base(ndarray arr): # <<<<<<<<<<<<<< @@ -5178,7 +5175,7 @@ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__py return __pyx_r; } -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1034 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1034 * # Versions of the import_* functions which are more suitable for * # Cython code. * cdef inline int import_array() except -1: # <<<<<<<<<<<<<< @@ -5199,7 +5196,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { PyObject *__pyx_t_8 = NULL; __Pyx_RefNannySetupContext("import_array", 0); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 * # Cython code. * cdef inline int import_array() except -1: * try: # <<<<<<<<<<<<<< @@ -5215,7 +5212,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { __Pyx_XGOTREF(__pyx_t_3); /*try:*/ { - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1036 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1036 * cdef inline int import_array() except -1: * try: * _import_array() # <<<<<<<<<<<<<< @@ -5224,7 +5221,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { */ __pyx_t_4 = _import_array(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 1036, __pyx_L3_error) - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 * # Cython code. * cdef inline int import_array() except -1: * try: # <<<<<<<<<<<<<< @@ -5238,7 +5235,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { goto __pyx_L8_try_end; __pyx_L3_error:; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1037 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1037 * try: * _import_array() * except Exception: # <<<<<<<<<<<<<< @@ -5253,7 +5250,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { __Pyx_GOTREF(__pyx_t_6); __Pyx_GOTREF(__pyx_t_7); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1038 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1038 * _import_array() * except Exception: * raise ImportError("numpy.core.multiarray failed to import") # <<<<<<<<<<<<<< @@ -5269,7 +5266,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1035 * # Cython code. * cdef inline int import_array() except -1: * try: # <<<<<<<<<<<<<< @@ -5284,7 +5281,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { __pyx_L8_try_end:; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1034 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1034 * # Versions of the import_* functions which are more suitable for * # Cython code. * cdef inline int import_array() except -1: # <<<<<<<<<<<<<< @@ -5307,7 +5304,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { return __pyx_r; } -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1040 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1040 * raise ImportError("numpy.core.multiarray failed to import") * * cdef inline int import_umath() except -1: # <<<<<<<<<<<<<< @@ -5328,7 +5325,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { PyObject *__pyx_t_8 = NULL; __Pyx_RefNannySetupContext("import_umath", 0); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 * * cdef inline int import_umath() except -1: * try: # <<<<<<<<<<<<<< @@ -5344,7 +5341,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { __Pyx_XGOTREF(__pyx_t_3); /*try:*/ { - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1042 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1042 * cdef inline int import_umath() except -1: * try: * _import_umath() # <<<<<<<<<<<<<< @@ -5353,7 +5350,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { */ __pyx_t_4 = _import_umath(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 1042, __pyx_L3_error) - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 * * cdef inline int import_umath() except -1: * try: # <<<<<<<<<<<<<< @@ -5367,7 +5364,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { goto __pyx_L8_try_end; __pyx_L3_error:; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1043 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1043 * try: * _import_umath() * except Exception: # <<<<<<<<<<<<<< @@ -5382,7 +5379,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { __Pyx_GOTREF(__pyx_t_6); __Pyx_GOTREF(__pyx_t_7); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1044 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1044 * _import_umath() * except Exception: * raise ImportError("numpy.core.umath failed to import") # <<<<<<<<<<<<<< @@ -5398,7 +5395,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1041 * * cdef inline int import_umath() except -1: * try: # <<<<<<<<<<<<<< @@ -5413,7 +5410,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { __pyx_L8_try_end:; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1040 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1040 * raise ImportError("numpy.core.multiarray failed to import") * * cdef inline int import_umath() except -1: # <<<<<<<<<<<<<< @@ -5436,7 +5433,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { return __pyx_r; } -/* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 * raise ImportError("numpy.core.umath failed to import") * * cdef inline int import_ufunc() except -1: # <<<<<<<<<<<<<< @@ -5457,7 +5454,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { PyObject *__pyx_t_8 = NULL; __Pyx_RefNannySetupContext("import_ufunc", 0); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 * * cdef inline int import_ufunc() except -1: * try: # <<<<<<<<<<<<<< @@ -5473,7 +5470,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { __Pyx_XGOTREF(__pyx_t_3); /*try:*/ { - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1048 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1048 * cdef inline int import_ufunc() except -1: * try: * _import_umath() # <<<<<<<<<<<<<< @@ -5482,7 +5479,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { */ __pyx_t_4 = _import_umath(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 1048, __pyx_L3_error) - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 * * cdef inline int import_ufunc() except -1: * try: # <<<<<<<<<<<<<< @@ -5496,7 +5493,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { goto __pyx_L8_try_end; __pyx_L3_error:; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1049 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1049 * try: * _import_umath() * except Exception: # <<<<<<<<<<<<<< @@ -5510,7 +5507,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { __Pyx_GOTREF(__pyx_t_6); __Pyx_GOTREF(__pyx_t_7); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1050 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1050 * _import_umath() * except Exception: * raise ImportError("numpy.core.umath failed to import") # <<<<<<<<<<<<<< @@ -5524,7 +5521,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1047 * * cdef inline int import_ufunc() except -1: * try: # <<<<<<<<<<<<<< @@ -5539,7 +5536,7 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { __pyx_L8_try_end:; } - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 * raise ImportError("numpy.core.umath failed to import") * * cdef inline int import_ufunc() except -1: # <<<<<<<<<<<<<< @@ -5883,7 +5880,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): * raise ValueError(u"ndarray is not C contiguous") # <<<<<<<<<<<<<< @@ -5894,7 +5891,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__3); __Pyx_GIVEREF(__pyx_tuple__3); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): * raise ValueError(u"ndarray is not Fortran contiguous") # <<<<<<<<<<<<<< @@ -5905,7 +5902,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__4); __Pyx_GIVEREF(__pyx_tuple__4); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 * if ((descr.byteorder == c'>' and little_endian) or * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< @@ -5916,7 +5913,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__5); __Pyx_GIVEREF(__pyx_tuple__5); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":856 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":856 * * if (end - f) - (new_offset - offset[0]) < 15: * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") # <<<<<<<<<<<<<< @@ -5927,7 +5924,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__6); __Pyx_GIVEREF(__pyx_tuple__6); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":880 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":880 * t = child.type_num * if end - f < 5: * raise RuntimeError(u"Format string allocated too short.") # <<<<<<<<<<<<<< @@ -5938,7 +5935,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__7); __Pyx_GIVEREF(__pyx_tuple__7); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1038 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1038 * _import_array() * except Exception: * raise ImportError("numpy.core.multiarray failed to import") # <<<<<<<<<<<<<< @@ -5949,7 +5946,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__8); __Pyx_GIVEREF(__pyx_tuple__8); - /* "../../envs/gensim27/local/lib/python2.7/site-packages/Cython/Includes/numpy/__init__.pxd":1044 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1044 * _import_umath() * except Exception: * raise ImportError("numpy.core.umath failed to import") # <<<<<<<<<<<<<< @@ -8978,7 +8975,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { theta = 0; } else { r = -a.real; - theta = atan2f(0.0, -1.0); + theta = atan2f(0, -1); } } else { r = __Pyx_c_abs_float(a); @@ -9133,7 +9130,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { theta = 0; } else { r = -a.real; - theta = atan2(0.0, -1.0); + theta = atan2(0, -1); } } else { r = __Pyx_c_abs_double(a); diff --git a/gensim/models/_utils_any2vec.pyx b/gensim/models/_utils_any2vec.pyx index 1cafacb6fc..fe66c129c4 100644 --- a/gensim/models/_utils_any2vec.pyx +++ b/gensim/models/_utils_any2vec.pyx @@ -152,6 +152,6 @@ cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): j += 1 if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): ngram = bytes(utf8_word[i:j]) - ngrams.append(bytes(ngram)) + ngrams.append(ngram) n += 1 return ngrams From bb7282d9e2e6065ed10844256be05d66fcdcd3f8 Mon Sep 17 00:00:00 2001 From: Ivan Menshikh Date: Wed, 23 Jan 2019 16:36:39 +0500 Subject: [PATCH 27/32] fix tests (wide unicode issue) --- gensim/test/test_utils.py | 48 +++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/gensim/test/test_utils.py b/gensim/test/test_utils.py index a71b828887..09bcae8ff1 100644 --- a/gensim/test/test_utils.py +++ b/gensim/test/test_utils.py @@ -8,6 +8,7 @@ """ from __future__ import unicode_literals +import sys import logging import unittest @@ -403,10 +404,6 @@ def setUp(self): 'тест': ['<те', 'тес', 'ест', 'ст>', '<тес', 'тест', 'ест>', '<тест', 'тест>'], 'テスト': ['<テス', 'テスト', 'スト>', '<テスト', 'テスト>', '<テスト>'], '試し': ['<試し', '試し>', '<試し>'], - '🚑🚒🚓🚕': [ - '<🚑🚒', '🚑🚒🚓', '🚒🚓🚕', '🚓🚕>', - '<🚑🚒🚓', '🚑🚒🚓🚕', '🚒🚓🚕>', '<🚑🚒🚓🚕', '🚑🚒🚓🚕>' - ], } self.expected_bytes = { 'test': [b'', b'est', b'est>', b'st>'], @@ -425,6 +422,15 @@ def setUp(self): b'\xe3\x83\x86\xe3\x82\xb9\xe3\x83\x88>', b'\xe3\x82\xb9\xe3\x83\x88>' ], '試し': [b'<\xe8\xa9\xa6\xe3\x81\x97', b'<\xe8\xa9\xa6\xe3\x81\x97>', b'\xe8\xa9\xa6\xe3\x81\x97>'], + } + + self.expected_text_wide_unicode = { + '🚑🚒🚓🚕': [ + '<🚑🚒', '🚑🚒🚓', '🚒🚓🚕', '🚓🚕>', + '<🚑🚒🚓', '🚑🚒🚓🚕', '🚒🚓🚕>', '<🚑🚒🚓🚕', '🚑🚒🚓🚕>' + ], + } + self.expected_bytes_wide_unicode = { '🚑🚒🚓🚕': [ b'<\xf0\x9f\x9a\x91\xf0\x9f\x9a\x92', b'<\xf0\x9f\x9a\x91\xf0\x9f\x9a\x92\xf0\x9f\x9a\x93', @@ -444,6 +450,13 @@ def test_text_py(self): actual = gensim.models.utils_any2vec._compute_ngrams_py(word, 3, 5) self.assertEqual(expected, actual) + @unittest.skipIf(sys.maxunicode == 0xffff, "Python interpreter doesn't support UCS-4 (wide unicode)") + def test_text_py_wide_unicode(self): + for word in self.expected_text_wide_unicode: + expected = self.expected_text_wide_unicode[word] + actual = gensim.models.utils_any2vec._compute_ngrams_py(word, 3, 5) + self.assertEqual(expected, actual) + @unittest.skipIf(DISABLE_CYTHON_TESTS, 'Cython functions are not properly compiled') def test_text_cy(self): for word in self.expected_text: @@ -451,6 +464,14 @@ def test_text_cy(self): actual = gensim.models.utils_any2vec._compute_ngrams_cy(word, 3, 5) self.assertEqual(expected, actual) + @unittest.skipIf(DISABLE_CYTHON_TESTS, 'Cython functions are not properly compiled') + @unittest.skipIf(sys.maxunicode == 0xffff, "Python interpreter doesn't support UCS-4 (wide unicode)") + def test_text_cy_wide_unicode(self): + for word in self.expected_text_wide_unicode: + expected = self.expected_text_wide_unicode[word] + actual = gensim.models.utils_any2vec._compute_ngrams_cy(word, 3, 5) + self.assertEqual(expected, actual) + def test_bytes_py(self): for word in self.expected_bytes: expected = self.expected_bytes[word] @@ -465,6 +486,16 @@ def test_bytes_py(self): # self.assertEqual(sorted(expected_text), sorted(actual_text)) + for word in self.expected_bytes_wide_unicode: + expected = self.expected_bytes_wide_unicode[word] + actual = gensim.models.utils_any2vec._compute_ngrams_bytes_py(word, 3, 5) + self.assertEqual(expected, actual) + + expected_text = self.expected_text_wide_unicode[word] + actual_text = [n.decode('utf-8') for n in actual] + + self.assertEqual(sorted(expected_text), sorted(actual_text)) + @unittest.skipIf(DISABLE_CYTHON_TESTS, 'Cython functions are not properly compiled') def test_bytes_cy(self): for word in self.expected_bytes: @@ -476,6 +507,15 @@ def test_bytes_cy(self): actual_text = [n.decode('utf-8') for n in actual] self.assertEqual(sorted(expected_text), sorted(actual_text)) + for word in self.expected_bytes_wide_unicode: + expected = self.expected_bytes_wide_unicode[word] + actual = gensim.models.utils_any2vec._compute_ngrams_bytes_cy(word, 3, 5) + self.assertEqual(expected, actual) + + expected_text = self.expected_text_wide_unicode[word] + actual_text = [n.decode('utf-8') for n in actual] + self.assertEqual(sorted(expected_text), sorted(actual_text)) + if __name__ == '__main__': logging.root.setLevel(logging.WARNING) From 07baf845d4dc2cb52344cde3ea3c5eead15406a5 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Wed, 23 Jan 2019 18:25:24 +1100 Subject: [PATCH 28/32] add test against actual FB implementation --- gensim/test/test_data/fb-ngrams.txt | 103 ++++++++++++++++++++++++++++ gensim/test/test_utils.py | 56 +++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 gensim/test/test_data/fb-ngrams.txt diff --git a/gensim/test/test_data/fb-ngrams.txt b/gensim/test/test_data/fb-ngrams.txt new file mode 100644 index 0000000000..bb22c620a3 --- /dev/null +++ b/gensim/test/test_data/fb-ngrams.txt @@ -0,0 +1,103 @@ + +test +test 0 0 0 0 0 + -0.058167 0.084801 -0.10452 0.085963 -0.14475 +tes 0.066033 0.011132 0.13224 0.19839 -0.037827 +test -0.12628 0.053912 0.051677 0.24844 0.083743 +test> -0.016497 0.1322 0.21391 0.20869 -0.044566 +est -0.12915 -0.02457 0.10145 0.23414 0.17121 +est> -0.1776 0.19906 -0.15572 0.16449 -0.12887 +st> -0.13712 0.028013 -0.041927 0.11244 0.12261 + + +at the +at the 0 0 0 0 0 + 0.17667 0.026172 -0.15457 0.081664 -0.1315 + th 0.1095 0.077397 0.052745 -0.043956 0.033436 + the -0.15504 0.11166 -0.16434 0.089162 -0.16199 + the> 0.097803 -0.054657 -0.028828 0.21607 0.11286 +the -0.046122 -0.071463 0.20759 0.25732 0.23174 +the> 0.04535 0.15192 -0.0021161 0.014343 0.12982 +he> -0.046122 -0.071463 0.20759 0.25732 0.23174 + + +atnthe +atnthe 0 0 0 0 0 + -0.065392 -0.085291 0.088378 0.024628 0.12715 +nth 0.067156 -0.068108 0.16394 0.18192 0.18902 +nthe 0.13526 -0.023216 -0.10788 0.24512 -0.022174 +nthe> -0.20296 -0.11805 0.20277 0.063027 -0.041727 +the -0.046122 -0.071463 0.20759 0.25732 0.23174 +the> 0.04535 0.15192 -0.0021161 0.014343 0.12982 +he> -0.046122 -0.071463 0.20759 0.25732 0.23174 + + +тест +тест 0 0 0 0 0 +<те -0.17079 -0.029842 0.10298 -0.12632 -0.14488 +<тес -0.026151 0.13071 0.20833 0.17143 0.040533 +<тест 0.17667 0.026172 -0.15457 0.081664 -0.1315 +<тест> -0.065392 -0.085291 0.088378 0.024628 0.12715 +тес -0.08172 -0.13699 0.14081 -0.036949 -0.037403 +тест -0.11209 0.068818 -0.137 -0.088165 0.13144 +тест> -0.20296 -0.11805 0.20277 0.063027 -0.041727 +ест -0.17079 -0.029842 0.10298 -0.12632 -0.14488 +ест> 0.16922 -0.1162 0.1462 0.1085 0.02952 +ст> -0.17261 0.086891 0.21772 0.10036 0.19114 + + +テスト +テスト 0 0 0 0 0 +<テス 0.066033 0.011132 0.13224 0.19839 -0.037827 +<テスト -0.12915 -0.02457 0.10145 0.23414 0.17121 +<テスト> -0.10241 0.0090735 0.097437 0.055768 0.03622 +テスト 0.048443 0.15183 -0.058779 -0.1095 0.10566 +テスト> -0.12628 0.053912 0.051677 0.24844 0.083743 +スト> 0.12972 0.18282 -0.18266 -0.072433 0.19318 + + +試し +試し 0 0 0 0 0 +<試し 0.1095 0.077397 0.052745 -0.043956 0.033436 +<試し> 0.11952 0.15093 -0.075759 0.18327 0.12471 +試し> -0.14656 0.072651 -0.16777 0.20026 0.21199 + + +🚑🚒🚓🚕 +🚑🚒🚓🚕 0 0 0 0 0 +<🚑🚒 -0.065364 0.17951 -0.17059 0.11436 -0.12783 +<🚑🚒🚓 -0.13712 0.028013 -0.041927 0.11244 0.12261 +<🚑🚒🚓🚕 0.18643 0.17268 0.024482 0.044628 -0.14716 +<🚑🚒🚓🚕> -0.17261 0.086891 0.21772 0.10036 0.19114 +🚑🚒🚓 0.11122 -0.16219 -0.15235 0.21407 0.14858 +🚑🚒🚓🚕 -0.15504 0.11166 -0.16434 0.089162 -0.16199 +🚑🚒🚓🚕> -0.13712 0.028013 -0.041927 0.11244 0.12261 +🚒🚓🚕 -0.20296 -0.11805 0.20277 0.063027 -0.041727 +🚒🚓🚕> 0.13526 -0.023216 -0.10788 0.24512 -0.022174 +🚓🚕> -0.1776 0.19906 -0.15572 0.16449 -0.12887 + diff --git a/gensim/test/test_utils.py b/gensim/test/test_utils.py index 09bcae8ff1..c23087580f 100644 --- a/gensim/test/test_utils.py +++ b/gensim/test/test_utils.py @@ -20,6 +20,8 @@ import gensim.models.utils_any2vec +import smart_open + DISABLE_CYTHON_TESTS = getattr(gensim.models.utils_any2vec, 'FAST_VERSION', None) == -1 @@ -516,6 +518,60 @@ def test_bytes_cy(self): actual_text = [n.decode('utf-8') for n in actual] self.assertEqual(sorted(expected_text), sorted(actual_text)) + def test_fb(self): + """Test against results from Facebook's implementation.""" + with smart_open.smart_open(datapath('fb-ngrams.txt'), 'r', encoding='utf-8') as fin: + fb = dict(_read_fb(fin)) + + for word, expected in fb.items(): + # + # The model was trained with minn=3, maxn=6 + # + actual = gensim.models.utils_any2vec._compute_ngrams_py(word, 3, 6) + self.assertEqual(sorted(expected), sorted(actual)) + + +def _read_fb(fin): + """Read ngrams from output of the FB utility.""" + # + # $ cat words.txt + # test + # at the + # at\nthe + # тест + # テスト + # 試し + # 🚑🚒🚓🚕 + # $ while read w; + # do + # echo ""; + # echo $w; + # ./fasttext print-ngrams gensim/test/test_data/crime-and-punishment.bin "$w"; + # echo ""; + # done < words.txt > gensim/test/test_data/fb-ngrams.txt + # + while fin: + line = fin.readline().rstrip() + if not line: + break + + assert line == '' + word = fin.readline().rstrip() + + fin.readline() # ignore this line, it contains an origin vector for the full term + + ngrams = [] + while True: + line = fin.readline().rstrip() + if line == '': + break + + columns = line.split(' ') + term = ' '.join(columns[:-5]) + ngrams.append(term) + + yield word, ngrams + if __name__ == '__main__': logging.root.setLevel(logging.WARNING) From ce1a63137336d3f9665fece4a6ac624e67ad528b Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Thu, 24 Jan 2019 11:53:36 +1100 Subject: [PATCH 29/32] adding temporary benchmarking code --- bench.py | 13 + gensim/models/_utils_any2vec.c | 641 +++++++++++++++++++++++++++++++ gensim/models/_utils_any2vec.pyx | 25 ++ 3 files changed, 679 insertions(+) create mode 100644 bench.py diff --git a/bench.py b/bench.py new file mode 100644 index 0000000000..dcec83ddb7 --- /dev/null +++ b/bench.py @@ -0,0 +1,13 @@ +import gensim.downloader as api +import itertools as it +from gensim.models.utils_any2vec import compute_ngrams_bytes, ft_hash_bytes + +words = tuple(it.chain.from_iterable(api.load("text8"))) +assert len(words) == 17005207 # long enough + +words = words[:100000] + + +def benchmark(words=words, ngram_func=compute_ngrams_bytes, hash_func=ft_hash_bytes): + for w in words: + arr = [hash_func(ngram) % 10000 for ngram in ngram_func(w, 3, 6)] diff --git a/gensim/models/_utils_any2vec.c b/gensim/models/_utils_any2vec.c index 7e7ca73364..8c9052ba50 100644 --- a/gensim/models/_utils_any2vec.c +++ b/gensim/models/_utils_any2vec.c @@ -1693,6 +1693,9 @@ static CYTHON_INLINE char __Pyx_PyInt_As_char(PyObject *); /* CIntFromPy.proto */ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); +/* CIntFromPy.proto */ +static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *); + /* CIntFromPy.proto */ static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); @@ -1753,6 +1756,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObjec static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__is_utf8_continue_py3(PyObject *); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__is_utf8_continue_py2(PyObject *); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(PyObject *, unsigned int, unsigned int, int __pyx_skip_dispatch); /*proto*/ +static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes_optimized(PyObject *, unsigned int, unsigned int, int __pyx_skip_dispatch); /*proto*/ static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*)(PyObject *)); /*proto*/ #define __Pyx_MODULE_NAME "gensim.models._utils_any2vec" extern int __pyx_module_is_main_gensim__models___utils_any2vec; @@ -1842,6 +1846,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(CYTHON_ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string); /* proto */ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_8compute_ngrams_bytes_optimized(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info); /* proto */ static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(PyObject *__pyx_self, PyObject *__pyx_v_b); /* proto */ @@ -3004,6 +3009,7 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P * ngrams.append(ngram) * n += 1 # <<<<<<<<<<<<<< * return ngrams + * */ __pyx_v_n = (__pyx_v_n + 1); } @@ -3014,6 +3020,8 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P * ngrams.append(ngram) * n += 1 * return ngrams # <<<<<<<<<<<<<< + * + * */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_ngrams); @@ -3141,6 +3149,449 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes return __pyx_r; } +/* "gensim/models/_utils_any2vec.pyx":160 + * + * + * cpdef compute_ngrams_bytes_optimized(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< + * cdef unsigned char _mb_mask = 0xC0 + * cdef unsigned char _mb_start = 0x80 + */ + +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_9compute_ngrams_bytes_optimized(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes_optimized(PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n, CYTHON_UNUSED int __pyx_skip_dispatch) { + unsigned char __pyx_v__mb_mask; + unsigned char __pyx_v__mb_start; + PyObject *__pyx_v_utf8_word = 0; + unsigned char const *__pyx_v_bytez; + size_t __pyx_v_num_bytes; + size_t __pyx_v_j; + size_t __pyx_v_i; + size_t __pyx_v_n; + PyObject *__pyx_v_ngrams = NULL; + PyObject *__pyx_v_ngram = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + unsigned char const *__pyx_t_4; + Py_ssize_t __pyx_t_5; + size_t __pyx_t_6; + size_t __pyx_t_7; + size_t __pyx_t_8; + int __pyx_t_9; + size_t __pyx_t_10; + size_t __pyx_t_11; + int __pyx_t_12; + int __pyx_t_13; + int __pyx_t_14; + __Pyx_RefNannySetupContext("compute_ngrams_bytes_optimized", 0); + + /* "gensim/models/_utils_any2vec.pyx":161 + * + * cpdef compute_ngrams_bytes_optimized(word, unsigned int min_n, unsigned int max_n): + * cdef unsigned char _mb_mask = 0xC0 # <<<<<<<<<<<<<< + * cdef unsigned char _mb_start = 0x80 + * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") + */ + __pyx_v__mb_mask = 0xC0; + + /* "gensim/models/_utils_any2vec.pyx":162 + * cpdef compute_ngrams_bytes_optimized(word, unsigned int min_n, unsigned int max_n): + * cdef unsigned char _mb_mask = 0xC0 + * cdef unsigned char _mb_start = 0x80 # <<<<<<<<<<<<<< + * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") + * cdef const unsigned char *bytez = utf8_word + */ + __pyx_v__mb_start = 0x80; + + /* "gensim/models/_utils_any2vec.pyx":163 + * cdef unsigned char _mb_mask = 0xC0 + * cdef unsigned char _mb_start = 0x80 + * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") # <<<<<<<<<<<<<< + * cdef const unsigned char *bytez = utf8_word + * cdef size_t num_bytes = len(utf8_word) + */ + __pyx_t_2 = __Pyx_PyString_FormatSafe(__pyx_kp_s_s, __pyx_v_word); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 163, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 163, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_2)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + } + } + __pyx_t_1 = (__pyx_t_2) ? __Pyx_PyObject_Call2Args(__pyx_t_3, __pyx_t_2, __pyx_kp_s_utf_8) : __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_kp_s_utf_8); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 163, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 163, __pyx_L1_error) + __pyx_v_utf8_word = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "gensim/models/_utils_any2vec.pyx":164 + * cdef unsigned char _mb_start = 0x80 + * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") + * cdef const unsigned char *bytez = utf8_word # <<<<<<<<<<<<<< + * cdef size_t num_bytes = len(utf8_word) + * cdef size_t j, i, n + */ + if (unlikely(__pyx_v_utf8_word == Py_None)) { + PyErr_SetString(PyExc_TypeError, "expected bytes, NoneType found"); + __PYX_ERR(0, 164, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_PyBytes_AsUString(__pyx_v_utf8_word); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) __PYX_ERR(0, 164, __pyx_L1_error) + __pyx_v_bytez = __pyx_t_4; + + /* "gensim/models/_utils_any2vec.pyx":165 + * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") + * cdef const unsigned char *bytez = utf8_word + * cdef size_t num_bytes = len(utf8_word) # <<<<<<<<<<<<<< + * cdef size_t j, i, n + * + */ + if (unlikely(__pyx_v_utf8_word == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 165, __pyx_L1_error) + } + __pyx_t_5 = PyBytes_GET_SIZE(__pyx_v_utf8_word); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 165, __pyx_L1_error) + __pyx_v_num_bytes = __pyx_t_5; + + /* "gensim/models/_utils_any2vec.pyx":168 + * cdef size_t j, i, n + * + * ngrams = [] # <<<<<<<<<<<<<< + * for i in range(num_bytes): + * if bytez[i] & _mb_mask == _mb_start: + */ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 168, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_ngrams = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "gensim/models/_utils_any2vec.pyx":169 + * + * ngrams = [] + * for i in range(num_bytes): # <<<<<<<<<<<<<< + * if bytez[i] & _mb_mask == _mb_start: + * continue + */ + __pyx_t_6 = __pyx_v_num_bytes; + __pyx_t_7 = __pyx_t_6; + for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) { + __pyx_v_i = __pyx_t_8; + + /* "gensim/models/_utils_any2vec.pyx":170 + * ngrams = [] + * for i in range(num_bytes): + * if bytez[i] & _mb_mask == _mb_start: # <<<<<<<<<<<<<< + * continue + * + */ + __pyx_t_9 = ((((__pyx_v_bytez[__pyx_v_i]) & __pyx_v__mb_mask) == __pyx_v__mb_start) != 0); + if (__pyx_t_9) { + + /* "gensim/models/_utils_any2vec.pyx":171 + * for i in range(num_bytes): + * if bytez[i] & _mb_mask == _mb_start: + * continue # <<<<<<<<<<<<<< + * + * j, n = i, 1 + */ + goto __pyx_L3_continue; + + /* "gensim/models/_utils_any2vec.pyx":170 + * ngrams = [] + * for i in range(num_bytes): + * if bytez[i] & _mb_mask == _mb_start: # <<<<<<<<<<<<<< + * continue + * + */ + } + + /* "gensim/models/_utils_any2vec.pyx":173 + * continue + * + * j, n = i, 1 # <<<<<<<<<<<<<< + * while j < num_bytes and n <= max_n: + * j += 1 + */ + __pyx_t_10 = __pyx_v_i; + __pyx_t_11 = 1; + __pyx_v_j = __pyx_t_10; + __pyx_v_n = __pyx_t_11; + + /* "gensim/models/_utils_any2vec.pyx":174 + * + * j, n = i, 1 + * while j < num_bytes and n <= max_n: # <<<<<<<<<<<<<< + * j += 1 + * while j < num_bytes and (bytez[j] & _mb_mask) == _mb_start: + */ + while (1) { + __pyx_t_12 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); + if (__pyx_t_12) { + } else { + __pyx_t_9 = __pyx_t_12; + goto __pyx_L8_bool_binop_done; + } + __pyx_t_12 = ((__pyx_v_n <= __pyx_v_max_n) != 0); + __pyx_t_9 = __pyx_t_12; + __pyx_L8_bool_binop_done:; + if (!__pyx_t_9) break; + + /* "gensim/models/_utils_any2vec.pyx":175 + * j, n = i, 1 + * while j < num_bytes and n <= max_n: + * j += 1 # <<<<<<<<<<<<<< + * while j < num_bytes and (bytez[j] & _mb_mask) == _mb_start: + * j += 1 + */ + __pyx_v_j = (__pyx_v_j + 1); + + /* "gensim/models/_utils_any2vec.pyx":176 + * while j < num_bytes and n <= max_n: + * j += 1 + * while j < num_bytes and (bytez[j] & _mb_mask) == _mb_start: # <<<<<<<<<<<<<< + * j += 1 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + */ + while (1) { + __pyx_t_12 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); + if (__pyx_t_12) { + } else { + __pyx_t_9 = __pyx_t_12; + goto __pyx_L12_bool_binop_done; + } + __pyx_t_12 = ((((__pyx_v_bytez[__pyx_v_j]) & __pyx_v__mb_mask) == __pyx_v__mb_start) != 0); + __pyx_t_9 = __pyx_t_12; + __pyx_L12_bool_binop_done:; + if (!__pyx_t_9) break; + + /* "gensim/models/_utils_any2vec.pyx":177 + * j += 1 + * while j < num_bytes and (bytez[j] & _mb_mask) == _mb_start: + * j += 1 # <<<<<<<<<<<<<< + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + * ngram = bytes(bytez[i:j]) + */ + __pyx_v_j = (__pyx_v_j + 1); + } + + /* "gensim/models/_utils_any2vec.pyx":178 + * while j < num_bytes and (bytez[j] & _mb_mask) == _mb_start: + * j += 1 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< + * ngram = bytes(bytez[i:j]) + * ngrams.append(ngram) + */ + __pyx_t_12 = ((__pyx_v_n >= __pyx_v_min_n) != 0); + if (__pyx_t_12) { + } else { + __pyx_t_9 = __pyx_t_12; + goto __pyx_L15_bool_binop_done; + } + __pyx_t_13 = ((__pyx_v_n == 1) != 0); + if (__pyx_t_13) { + } else { + __pyx_t_12 = __pyx_t_13; + goto __pyx_L17_bool_binop_done; + } + __pyx_t_13 = ((__pyx_v_i == 0) != 0); + if (!__pyx_t_13) { + } else { + __pyx_t_12 = __pyx_t_13; + goto __pyx_L17_bool_binop_done; + } + __pyx_t_13 = ((__pyx_v_j == __pyx_v_num_bytes) != 0); + __pyx_t_12 = __pyx_t_13; + __pyx_L17_bool_binop_done:; + __pyx_t_13 = ((!__pyx_t_12) != 0); + __pyx_t_9 = __pyx_t_13; + __pyx_L15_bool_binop_done:; + if (__pyx_t_9) { + + /* "gensim/models/_utils_any2vec.pyx":179 + * j += 1 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + * ngram = bytes(bytez[i:j]) # <<<<<<<<<<<<<< + * ngrams.append(ngram) + * n += 1 + */ + __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(((const char*)__pyx_v_bytez) + __pyx_v_i, __pyx_v_j - __pyx_v_i); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 179, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 179, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF_SET(__pyx_v_ngram, ((PyObject*)__pyx_t_3)); + __pyx_t_3 = 0; + + /* "gensim/models/_utils_any2vec.pyx":180 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + * ngram = bytes(bytez[i:j]) + * ngrams.append(ngram) # <<<<<<<<<<<<<< + * n += 1 + * return ngrams + */ + __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_v_ngram); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 180, __pyx_L1_error) + + /* "gensim/models/_utils_any2vec.pyx":178 + * while j < num_bytes and (bytez[j] & _mb_mask) == _mb_start: + * j += 1 + * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< + * ngram = bytes(bytez[i:j]) + * ngrams.append(ngram) + */ + } + + /* "gensim/models/_utils_any2vec.pyx":181 + * ngram = bytes(bytez[i:j]) + * ngrams.append(ngram) + * n += 1 # <<<<<<<<<<<<<< + * return ngrams + */ + __pyx_v_n = (__pyx_v_n + 1); + } + __pyx_L3_continue:; + } + + /* "gensim/models/_utils_any2vec.pyx":182 + * ngrams.append(ngram) + * n += 1 + * return ngrams # <<<<<<<<<<<<<< + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_ngrams); + __pyx_r = __pyx_v_ngrams; + goto __pyx_L0; + + /* "gensim/models/_utils_any2vec.pyx":160 + * + * + * cpdef compute_ngrams_bytes_optimized(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< + * cdef unsigned char _mb_mask = 0xC0 + * cdef unsigned char _mb_start = 0x80 + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes_optimized", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_utf8_word); + __Pyx_XDECREF(__pyx_v_ngrams); + __Pyx_XDECREF(__pyx_v_ngram); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_9compute_ngrams_bytes_optimized(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_6gensim_6models_14_utils_any2vec_8compute_ngrams_bytes_optimized[] = "compute_ngrams_bytes_optimized(word, unsigned int min_n, unsigned int max_n)"; +static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_9compute_ngrams_bytes_optimized(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_word = 0; + unsigned int __pyx_v_min_n; + unsigned int __pyx_v_max_n; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("compute_ngrams_bytes_optimized (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_word,&__pyx_n_s_min_n,&__pyx_n_s_max_n,0}; + PyObject* values[3] = {0,0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_word)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_min_n)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes_optimized", 1, 3, 3, 1); __PYX_ERR(0, 160, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 2: + if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_n)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes_optimized", 1, 3, 3, 2); __PYX_ERR(0, 160, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams_bytes_optimized") < 0)) __PYX_ERR(0, 160, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + } + __pyx_v_word = values[0]; + __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 160, __pyx_L3_error) + __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 160, __pyx_L3_error) + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes_optimized", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 160, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes_optimized", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_8compute_ngrams_bytes_optimized(__pyx_self, __pyx_v_word, __pyx_v_min_n, __pyx_v_max_n); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_8compute_ngrams_bytes_optimized(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("compute_ngrams_bytes_optimized", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes_optimized(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 160, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes_optimized", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 * # experimental exception made for __getbuffer__ and __releasebuffer__ * # -- the details of this may change. @@ -5782,6 +6233,7 @@ static PyMethodDef __pyx_methods[] = { {"ft_hash_broken", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_2ft_hash_broken}, {"compute_ngrams", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams, METH_VARARGS|METH_KEYWORDS, __pyx_doc_6gensim_6models_14_utils_any2vec_4compute_ngrams}, {"compute_ngrams_bytes", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes, METH_VARARGS|METH_KEYWORDS, __pyx_doc_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes}, + {"compute_ngrams_bytes_optimized", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_6gensim_6models_14_utils_any2vec_9compute_ngrams_bytes_optimized, METH_VARARGS|METH_KEYWORDS, __pyx_doc_6gensim_6models_14_utils_any2vec_8compute_ngrams_bytes_optimized}, {0, 0, 0, 0} }; @@ -9744,6 +10196,195 @@ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { return (int) -1; } +/* CIntFromPy */ +static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { + const size_t neg_one = (size_t) ((size_t) 0 - (size_t) 1), const_zero = (size_t) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(size_t) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(size_t, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (size_t) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (size_t) 0; + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, digits[0]) + case 2: + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 2 * PyLong_SHIFT) { + return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 3 * PyLong_SHIFT) { + return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 4 * PyLong_SHIFT) { + return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (size_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(size_t) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (size_t) 0; + case -1: __PYX_VERIFY_RETURN_INT(size_t, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, +digits[0]) + case -2: + if (8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + } +#endif + if (sizeof(size_t) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(size_t) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + size_t val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (size_t) -1; + } + } else { + size_t val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (size_t) -1; + val = __Pyx_PyInt_As_size_t(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to size_t"); + return (size_t) -1; +} + /* CIntFromPy */ static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { const long neg_one = (long) ((long) 0 - (long) 1), const_zero = (long) 0; diff --git a/gensim/models/_utils_any2vec.pyx b/gensim/models/_utils_any2vec.pyx index fe66c129c4..9c5965cd4b 100644 --- a/gensim/models/_utils_any2vec.pyx +++ b/gensim/models/_utils_any2vec.pyx @@ -155,3 +155,28 @@ cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): ngrams.append(ngram) n += 1 return ngrams + + +cpdef compute_ngrams_bytes_optimized(word, unsigned int min_n, unsigned int max_n): + cdef unsigned char _mb_mask = 0xC0 + cdef unsigned char _mb_start = 0x80 + cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") + cdef const unsigned char *bytez = utf8_word + cdef size_t num_bytes = len(utf8_word) + cdef size_t j, i, n + + ngrams = [] + for i in range(num_bytes): + if bytez[i] & _mb_mask == _mb_start: + continue + + j, n = i, 1 + while j < num_bytes and n <= max_n: + j += 1 + while j < num_bytes and (bytez[j] & _mb_mask) == _mb_start: + j += 1 + if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + ngram = bytes(bytez[i:j]) + ngrams.append(ngram) + n += 1 + return ngrams From e87ca05e17fee6d843b66224b1185d0e9e65c6d2 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Thu, 24 Jan 2019 12:43:54 +1100 Subject: [PATCH 30/32] replacing non-optimized code with optimized code --- gensim/models/_utils_any2vec.c | 4432 +++++++++--------------------- gensim/models/_utils_any2vec.pyx | 43 +- 2 files changed, 1309 insertions(+), 3166 deletions(-) diff --git a/gensim/models/_utils_any2vec.c b/gensim/models/_utils_any2vec.c index 8c9052ba50..1e21639afa 100644 --- a/gensim/models/_utils_any2vec.c +++ b/gensim/models/_utils_any2vec.c @@ -844,7 +844,6 @@ static const char *__pyx_filename; static const char *__pyx_f[] = { "gensim/models/_utils_any2vec.pyx", "__init__.pxd", - "stringsource", "type.pxd", }; @@ -1062,7 +1061,6 @@ static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(do /*--- Type declarations ---*/ -struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py; /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":815 * ctypedef npy_longdouble longdouble_t @@ -1100,19 +1098,6 @@ typedef npy_clongdouble __pyx_t_5numpy_clongdouble_t; */ typedef npy_cdouble __pyx_t_5numpy_complex_t; -/* "cfunc.to_py":64 - * - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< - * def wrap(object b): - * """wrap(b)""" - */ -struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py { - PyObject_HEAD - PyObject *(*__pyx_v_f)(PyObject *); -}; - - /* --- Runtime support code (head) --- */ /* Refnanny.proto */ #ifndef CYTHON_REFNANNY @@ -1286,27 +1271,6 @@ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ const char* function_name); -/* GetModuleGlobalName.proto */ -#if CYTHON_USE_DICT_VERSIONS -#define __Pyx_GetModuleGlobalName(var, name) {\ - static PY_UINT64_T __pyx_dict_version = 0;\ - static PyObject *__pyx_dict_cached_value = NULL;\ - (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_d))) ?\ - (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ - __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ -} -#define __Pyx_GetModuleGlobalNameUncached(var, name) {\ - PY_UINT64_T __pyx_dict_version;\ - PyObject *__pyx_dict_cached_value;\ - (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ -} -static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); -#else -#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) -#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) -static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); -#endif - /* PyCFunctionFastCall.proto */ #if CYTHON_FAST_PYCCALL static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); @@ -1348,28 +1312,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject /* PyObjectCallOneArg.proto */ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); -/* GetItemInt.proto */ -#define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ - (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ - __Pyx_GetItemInt_Fast(o, (Py_ssize_t)i, is_list, wraparound, boundscheck) :\ - (is_list ? (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL) :\ - __Pyx_GetItemInt_Generic(o, to_py_func(i)))) -#define __Pyx_GetItemInt_List(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ - (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ - __Pyx_GetItemInt_List_Fast(o, (Py_ssize_t)i, wraparound, boundscheck) :\ - (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) -static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, - int wraparound, int boundscheck); -#define __Pyx_GetItemInt_Tuple(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ - (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ - __Pyx_GetItemInt_Tuple_Fast(o, (Py_ssize_t)i, wraparound, boundscheck) :\ - (PyErr_SetString(PyExc_IndexError, "tuple index out of range"), (PyObject*)NULL)) -static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, - int wraparound, int boundscheck); -static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j); -static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, - int is_list, int wraparound, int boundscheck); - /* PyThreadStateGet.proto */ #if CYTHON_FAST_THREAD_STATE #define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; @@ -1464,70 +1406,6 @@ static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); #endif -/* FetchCommonType.proto */ -static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); - -/* CythonFunction.proto */ -#define __Pyx_CyFunction_USED 1 -#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 -#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 -#define __Pyx_CYFUNCTION_CCLASS 0x04 -#define __Pyx_CyFunction_GetClosure(f)\ - (((__pyx_CyFunctionObject *) (f))->func_closure) -#define __Pyx_CyFunction_GetClassObj(f)\ - (((__pyx_CyFunctionObject *) (f))->func_classobj) -#define __Pyx_CyFunction_Defaults(type, f)\ - ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) -#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ - ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) -typedef struct { - PyCFunctionObject func; -#if PY_VERSION_HEX < 0x030500A0 - PyObject *func_weakreflist; -#endif - PyObject *func_dict; - PyObject *func_name; - PyObject *func_qualname; - PyObject *func_doc; - PyObject *func_globals; - PyObject *func_code; - PyObject *func_closure; - PyObject *func_classobj; - void *defaults; - int defaults_pyobjects; - int flags; - PyObject *defaults_tuple; - PyObject *defaults_kwdict; - PyObject *(*defaults_getter)(PyObject *); - PyObject *func_annotations; -} __pyx_CyFunctionObject; -static PyTypeObject *__pyx_CyFunctionType = 0; -#define __Pyx_CyFunction_Check(obj) (__Pyx_TypeCheck(obj, __pyx_CyFunctionType)) -#define __Pyx_CyFunction_NewEx(ml, flags, qualname, self, module, globals, code)\ - __Pyx_CyFunction_New(__pyx_CyFunctionType, ml, flags, qualname, self, module, globals, code) -static PyObject *__Pyx_CyFunction_New(PyTypeObject *, PyMethodDef *ml, - int flags, PyObject* qualname, - PyObject *self, - PyObject *module, PyObject *globals, - PyObject* code); -static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *m, - size_t size, - int pyobjects); -static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, - PyObject *tuple); -static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, - PyObject *dict); -static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, - PyObject *dict); -static int __pyx_CyFunction_init(void); - -/* PyObject_GenericGetAttrNoDict.proto */ -#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 -static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name); -#else -#define __Pyx_PyObject_GenericGetAttrNoDict PyObject_GenericGetAttr -#endif - /* TypeImport.proto */ #ifndef __PYX_HAVE_RT_ImportType_proto #define __PYX_HAVE_RT_ImportType_proto @@ -1577,12 +1455,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint32_t(uint32_t value); /* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value); -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); - -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); - /* RealImag.proto */ #if CYTHON_CCOMPLEX #ifdef __cplusplus @@ -1681,6 +1553,9 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); #endif #endif +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); + /* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_enum__NPY_TYPES(enum NPY_TYPES value); @@ -1691,10 +1566,13 @@ static CYTHON_INLINE unsigned int __Pyx_PyInt_As_unsigned_int(PyObject *); static CYTHON_INLINE char __Pyx_PyInt_As_char(PyObject *); /* CIntFromPy.proto */ -static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); +static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *); /* CIntFromPy.proto */ -static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *); +static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); /* CIntFromPy.proto */ static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); @@ -1749,15 +1627,12 @@ static PyTypeObject *__pyx_ptype_5numpy_ufunc = 0; static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *, char *, char *, int *); /*proto*/ /* Module declarations from 'gensim.models._utils_any2vec' */ -static PyTypeObject *__pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py = 0; +static unsigned char __pyx_v_6gensim_6models_14_utils_any2vec__MB_MASK; +static unsigned char __pyx_v_6gensim_6models_14_utils_any2vec__MB_START; static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_bytes(PyObject *, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_ft_hash_broken(PyObject *, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams(PyObject *, unsigned int, unsigned int, int __pyx_skip_dispatch); /*proto*/ -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__is_utf8_continue_py3(PyObject *); /*proto*/ -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__is_utf8_continue_py2(PyObject *); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(PyObject *, unsigned int, unsigned int, int __pyx_skip_dispatch); /*proto*/ -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes_optimized(PyObject *, unsigned int, unsigned int, int __pyx_skip_dispatch); /*proto*/ -static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*)(PyObject *)); /*proto*/ #define __Pyx_MODULE_NAME "gensim.models._utils_any2vec" extern int __pyx_module_is_main_gensim__models___utils_any2vec; int __pyx_module_is_main_gensim__models___utils_any2vec = 0; @@ -1768,7 +1643,6 @@ static PyObject *__pyx_builtin_ValueError; static PyObject *__pyx_builtin_RuntimeError; static PyObject *__pyx_builtin_ImportError; static const char __pyx_k_[] = "<"; -static const char __pyx_k_b[] = "b"; static const char __pyx_k_s[] = "<%s>"; static const char __pyx_k__2[] = ">"; static const char __pyx_k_np[] = "np"; @@ -1778,7 +1652,6 @@ static const char __pyx_k_main[] = "__main__"; static const char __pyx_k_name[] = "__name__"; static const char __pyx_k_test[] = "__test__"; static const char __pyx_k_word[] = "word"; -static const char __pyx_k_wrap[] = "wrap"; static const char __pyx_k_max_n[] = "max_n"; static const char __pyx_k_min_n[] = "min_n"; static const char __pyx_k_numpy[] = "numpy"; @@ -1786,17 +1659,11 @@ static const char __pyx_k_range[] = "range"; static const char __pyx_k_utf_8[] = "utf-8"; static const char __pyx_k_encode[] = "encode"; static const char __pyx_k_import[] = "__import__"; -static const char __pyx_k_MB_MASK[] = "_MB_MASK"; -static const char __pyx_k_MB_START[] = "_MB_START"; static const char __pyx_k_ValueError[] = "ValueError"; static const char __pyx_k_ImportError[] = "ImportError"; -static const char __pyx_k_cfunc_to_py[] = "cfunc.to_py"; static const char __pyx_k_RuntimeError[] = "RuntimeError"; -static const char __pyx_k_stringsource[] = "stringsource"; -static const char __pyx_k_is_utf8_continue[] = "_is_utf8_continue"; static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; static const char __pyx_k_ndarray_is_not_C_contiguous[] = "ndarray is not C contiguous"; -static const char __pyx_k_Pyx_CFunc_object____object___t[] = "__Pyx_CFunc_object____object___to_py..wrap"; static const char __pyx_k_numpy_core_multiarray_failed_to[] = "numpy.core.multiarray failed to import"; static const char __pyx_k_unknown_dtype_code_in_numpy_pxd[] = "unknown dtype code in numpy.pxd (%d)"; static const char __pyx_k_Format_string_allocated_too_shor[] = "Format string allocated too short, see comment in numpy.pxd"; @@ -1809,20 +1676,14 @@ static PyObject *__pyx_kp_u_; static PyObject *__pyx_kp_u_Format_string_allocated_too_shor; static PyObject *__pyx_kp_u_Format_string_allocated_too_shor_2; static PyObject *__pyx_n_s_ImportError; -static PyObject *__pyx_n_s_MB_MASK; -static PyObject *__pyx_n_s_MB_START; static PyObject *__pyx_kp_u_Non_native_byte_order_not_suppor; static PyObject *__pyx_n_s_PY2; -static PyObject *__pyx_n_s_Pyx_CFunc_object____object___t; static PyObject *__pyx_n_s_RuntimeError; static PyObject *__pyx_n_s_ValueError; static PyObject *__pyx_kp_u__2; -static PyObject *__pyx_n_s_b; -static PyObject *__pyx_n_s_cfunc_to_py; static PyObject *__pyx_n_s_cline_in_traceback; static PyObject *__pyx_n_s_encode; static PyObject *__pyx_n_s_import; -static PyObject *__pyx_n_s_is_utf8_continue; static PyObject *__pyx_n_s_main; static PyObject *__pyx_n_s_max_n; static PyObject *__pyx_n_s_min_n; @@ -1836,25 +1697,18 @@ static PyObject *__pyx_kp_s_numpy_core_umath_failed_to_impor; static PyObject *__pyx_n_s_range; static PyObject *__pyx_kp_s_s; static PyObject *__pyx_n_s_six; -static PyObject *__pyx_kp_s_stringsource; static PyObject *__pyx_n_s_test; static PyObject *__pyx_kp_u_unknown_dtype_code_in_numpy_pxd; static PyObject *__pyx_kp_s_utf_8; static PyObject *__pyx_n_s_word; -static PyObject *__pyx_n_s_wrap; static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_ft_hash_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_bytez); /* proto */ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_2ft_hash_broken(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_string); /* proto */ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ -static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_8compute_ngrams_bytes_optimized(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n); /* proto */ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info); /* proto */ -static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(PyObject *__pyx_self, PyObject *__pyx_v_b); /* proto */ -static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_int_0; static PyObject *__pyx_int_1; -static PyObject *__pyx_int_128; -static PyObject *__pyx_int_192; static PyObject *__pyx_tuple__3; static PyObject *__pyx_tuple__4; static PyObject *__pyx_tuple__5; @@ -1862,8 +1716,6 @@ static PyObject *__pyx_tuple__6; static PyObject *__pyx_tuple__7; static PyObject *__pyx_tuple__8; static PyObject *__pyx_tuple__9; -static PyObject *__pyx_tuple__10; -static PyObject *__pyx_codeobj__11; /* Late includes */ /* "gensim/models/_utils_any2vec.pyx":22 @@ -2569,129 +2421,6 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_4compute_ngrams(CYTHO } /* "gensim/models/_utils_any2vec.pyx":104 - * - * - * cdef _is_utf8_continue_py3(b): # <<<<<<<<<<<<<< - * return b & _MB_MASK == _MB_START - * - */ - -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__is_utf8_continue_py3(PyObject *__pyx_v_b) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - __Pyx_RefNannySetupContext("_is_utf8_continue_py3", 0); - - /* "gensim/models/_utils_any2vec.pyx":105 - * - * cdef _is_utf8_continue_py3(b): - * return b & _MB_MASK == _MB_START # <<<<<<<<<<<<<< - * - * - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 105, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyNumber_And(__pyx_v_b, __pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 105, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 105, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = PyObject_RichCompare(__pyx_t_2, __pyx_t_1, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 105, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_r = __pyx_t_3; - __pyx_t_3 = 0; - goto __pyx_L0; - - /* "gensim/models/_utils_any2vec.pyx":104 - * - * - * cdef _is_utf8_continue_py3(b): # <<<<<<<<<<<<<< - * return b & _MB_MASK == _MB_START - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("gensim.models._utils_any2vec._is_utf8_continue_py3", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "gensim/models/_utils_any2vec.pyx":108 - * - * - * cdef _is_utf8_continue_py2(b): # <<<<<<<<<<<<<< - * return ord(b) & _MB_MASK == _MB_START - * - */ - -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__is_utf8_continue_py2(PyObject *__pyx_v_b) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - long __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - __Pyx_RefNannySetupContext("_is_utf8_continue_py2", 0); - - /* "gensim/models/_utils_any2vec.pyx":109 - * - * cdef _is_utf8_continue_py2(b): - * return ord(b) & _MB_MASK == _MB_START # <<<<<<<<<<<<<< - * - * - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_Ord(__pyx_v_b); if (unlikely(__pyx_t_1 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(0, 109, __pyx_L1_error) - __pyx_t_2 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 109, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_MB_MASK); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 109, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyNumber_And(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 109, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_MB_START); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 109, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_2 = PyObject_RichCompare(__pyx_t_4, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 109, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; - goto __pyx_L0; - - /* "gensim/models/_utils_any2vec.pyx":108 - * - * - * cdef _is_utf8_continue_py2(b): # <<<<<<<<<<<<<< - * return ord(b) & _MB_MASK == _MB_START - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("gensim.models._utils_any2vec._is_utf8_continue_py2", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "gensim/models/_utils_any2vec.pyx":115 * * * cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -2702,10 +2431,11 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec__is_utf8_continue_py2( static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n, CYTHON_UNUSED int __pyx_skip_dispatch) { PyObject *__pyx_v_utf8_word = 0; - int __pyx_v_num_bytes; - int __pyx_v_j; - int __pyx_v_i; - int __pyx_v_n; + unsigned char const *__pyx_v_bytez; + size_t __pyx_v_num_bytes; + size_t __pyx_v_j; + size_t __pyx_v_i; + size_t __pyx_v_n; PyObject *__pyx_v_ngrams = NULL; PyObject *__pyx_v_ngram = NULL; PyObject *__pyx_r = NULL; @@ -2713,29 +2443,29 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; - Py_ssize_t __pyx_t_4; - int __pyx_t_5; - int __pyx_t_6; - int __pyx_t_7; - PyObject *__pyx_t_8 = NULL; + unsigned char const *__pyx_t_4; + Py_ssize_t __pyx_t_5; + size_t __pyx_t_6; + size_t __pyx_t_7; + size_t __pyx_t_8; int __pyx_t_9; - int __pyx_t_10; - int __pyx_t_11; + size_t __pyx_t_10; + size_t __pyx_t_11; int __pyx_t_12; int __pyx_t_13; int __pyx_t_14; __Pyx_RefNannySetupContext("compute_ngrams_bytes", 0); - /* "gensim/models/_utils_any2vec.pyx":139 + /* "gensim/models/_utils_any2vec.pyx":128 * * """ * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") # <<<<<<<<<<<<<< - * cdef int num_bytes = len(utf8_word) - * cdef int j, i, n + * cdef const unsigned char *bytez = utf8_word + * cdef size_t num_bytes = len(utf8_word) */ - __pyx_t_2 = __Pyx_PyString_FormatSafe(__pyx_kp_s_s, __pyx_v_word); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 139, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyString_FormatSafe(__pyx_kp_s_s, __pyx_v_word); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 128, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 139, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 128, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = NULL; @@ -2750,101 +2480,94 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P } __pyx_t_1 = (__pyx_t_2) ? __Pyx_PyObject_Call2Args(__pyx_t_3, __pyx_t_2, __pyx_kp_s_utf_8) : __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_kp_s_utf_8); __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; - if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 139, __pyx_L1_error) + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 128, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 139, __pyx_L1_error) + if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 128, __pyx_L1_error) __pyx_v_utf8_word = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":140 + /* "gensim/models/_utils_any2vec.pyx":129 * """ * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") - * cdef int num_bytes = len(utf8_word) # <<<<<<<<<<<<<< - * cdef int j, i, n + * cdef const unsigned char *bytez = utf8_word # <<<<<<<<<<<<<< + * cdef size_t num_bytes = len(utf8_word) + * cdef size_t j, i, n + */ + if (unlikely(__pyx_v_utf8_word == Py_None)) { + PyErr_SetString(PyExc_TypeError, "expected bytes, NoneType found"); + __PYX_ERR(0, 129, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_PyBytes_AsUString(__pyx_v_utf8_word); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) __PYX_ERR(0, 129, __pyx_L1_error) + __pyx_v_bytez = __pyx_t_4; + + /* "gensim/models/_utils_any2vec.pyx":130 + * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") + * cdef const unsigned char *bytez = utf8_word + * cdef size_t num_bytes = len(utf8_word) # <<<<<<<<<<<<<< + * cdef size_t j, i, n * */ if (unlikely(__pyx_v_utf8_word == Py_None)) { PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); - __PYX_ERR(0, 140, __pyx_L1_error) + __PYX_ERR(0, 130, __pyx_L1_error) } - __pyx_t_4 = PyBytes_GET_SIZE(__pyx_v_utf8_word); if (unlikely(__pyx_t_4 == ((Py_ssize_t)-1))) __PYX_ERR(0, 140, __pyx_L1_error) - __pyx_v_num_bytes = __pyx_t_4; + __pyx_t_5 = PyBytes_GET_SIZE(__pyx_v_utf8_word); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 130, __pyx_L1_error) + __pyx_v_num_bytes = __pyx_t_5; - /* "gensim/models/_utils_any2vec.pyx":143 - * cdef int j, i, n + /* "gensim/models/_utils_any2vec.pyx":133 + * cdef size_t j, i, n * * ngrams = [] # <<<<<<<<<<<<<< * for i in range(num_bytes): - * if _is_utf8_continue(utf8_word[i]): + * if bytez[i] & _MB_MASK == _MB_START: */ - __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 143, __pyx_L1_error) + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 133, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_ngrams = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "gensim/models/_utils_any2vec.pyx":144 + /* "gensim/models/_utils_any2vec.pyx":134 * * ngrams = [] * for i in range(num_bytes): # <<<<<<<<<<<<<< - * if _is_utf8_continue(utf8_word[i]): + * if bytez[i] & _MB_MASK == _MB_START: * continue */ - __pyx_t_5 = __pyx_v_num_bytes; - __pyx_t_6 = __pyx_t_5; - for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_6; __pyx_t_7+=1) { - __pyx_v_i = __pyx_t_7; + __pyx_t_6 = __pyx_v_num_bytes; + __pyx_t_7 = __pyx_t_6; + for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) { + __pyx_v_i = __pyx_t_8; - /* "gensim/models/_utils_any2vec.pyx":145 + /* "gensim/models/_utils_any2vec.pyx":135 * ngrams = [] * for i in range(num_bytes): - * if _is_utf8_continue(utf8_word[i]): # <<<<<<<<<<<<<< + * if bytez[i] & _MB_MASK == _MB_START: # <<<<<<<<<<<<<< * continue * */ - __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_is_utf8_continue); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 145, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 145, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_8 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_3))) { - __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_3); - if (likely(__pyx_t_8)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); - __Pyx_INCREF(__pyx_t_8); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_3, function); - } - } - __pyx_t_1 = (__pyx_t_8) ? __Pyx_PyObject_Call2Args(__pyx_t_3, __pyx_t_8, __pyx_t_2) : __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); - __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 145, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_9 < 0)) __PYX_ERR(0, 145, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_9 = ((((__pyx_v_bytez[__pyx_v_i]) & __pyx_v_6gensim_6models_14_utils_any2vec__MB_MASK) == __pyx_v_6gensim_6models_14_utils_any2vec__MB_START) != 0); if (__pyx_t_9) { - /* "gensim/models/_utils_any2vec.pyx":146 + /* "gensim/models/_utils_any2vec.pyx":136 * for i in range(num_bytes): - * if _is_utf8_continue(utf8_word[i]): + * if bytez[i] & _MB_MASK == _MB_START: * continue # <<<<<<<<<<<<<< * * j, n = i, 1 */ goto __pyx_L3_continue; - /* "gensim/models/_utils_any2vec.pyx":145 + /* "gensim/models/_utils_any2vec.pyx":135 * ngrams = [] * for i in range(num_bytes): - * if _is_utf8_continue(utf8_word[i]): # <<<<<<<<<<<<<< + * if bytez[i] & _MB_MASK == _MB_START: # <<<<<<<<<<<<<< * continue * */ } - /* "gensim/models/_utils_any2vec.pyx":148 + /* "gensim/models/_utils_any2vec.pyx":138 * continue * * j, n = i, 1 # <<<<<<<<<<<<<< @@ -2856,12 +2579,12 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_v_j = __pyx_t_10; __pyx_v_n = __pyx_t_11; - /* "gensim/models/_utils_any2vec.pyx":149 + /* "gensim/models/_utils_any2vec.pyx":139 * * j, n = i, 1 * while j < num_bytes and n <= max_n: # <<<<<<<<<<<<<< * j += 1 - * while j < num_bytes and _is_utf8_continue(utf8_word[j]): + * while j < num_bytes and (bytez[j] & _MB_MASK) == _MB_START: */ while (1) { __pyx_t_12 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); @@ -2875,19 +2598,19 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_L8_bool_binop_done:; if (!__pyx_t_9) break; - /* "gensim/models/_utils_any2vec.pyx":150 + /* "gensim/models/_utils_any2vec.pyx":140 * j, n = i, 1 * while j < num_bytes and n <= max_n: * j += 1 # <<<<<<<<<<<<<< - * while j < num_bytes and _is_utf8_continue(utf8_word[j]): + * while j < num_bytes and (bytez[j] & _MB_MASK) == _MB_START: * j += 1 */ __pyx_v_j = (__pyx_v_j + 1); - /* "gensim/models/_utils_any2vec.pyx":151 + /* "gensim/models/_utils_any2vec.pyx":141 * while j < num_bytes and n <= max_n: * j += 1 - * while j < num_bytes and _is_utf8_continue(utf8_word[j]): # <<<<<<<<<<<<<< + * while j < num_bytes and (bytez[j] & _MB_MASK) == _MB_START: # <<<<<<<<<<<<<< * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): */ @@ -2898,47 +2621,26 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_t_9 = __pyx_t_12; goto __pyx_L12_bool_binop_done; } - __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_is_utf8_continue); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 151, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_utf8_word, __pyx_v_j, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 151, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_8 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_3))) { - __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_3); - if (likely(__pyx_t_8)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); - __Pyx_INCREF(__pyx_t_8); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_3, function); - } - } - __pyx_t_1 = (__pyx_t_8) ? __Pyx_PyObject_Call2Args(__pyx_t_3, __pyx_t_8, __pyx_t_2) : __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); - __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 151, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_12 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_12 < 0)) __PYX_ERR(0, 151, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_12 = ((((__pyx_v_bytez[__pyx_v_j]) & __pyx_v_6gensim_6models_14_utils_any2vec__MB_MASK) == __pyx_v_6gensim_6models_14_utils_any2vec__MB_START) != 0); __pyx_t_9 = __pyx_t_12; __pyx_L12_bool_binop_done:; if (!__pyx_t_9) break; - /* "gensim/models/_utils_any2vec.pyx":152 + /* "gensim/models/_utils_any2vec.pyx":142 * j += 1 - * while j < num_bytes and _is_utf8_continue(utf8_word[j]): + * while j < num_bytes and (bytez[j] & _MB_MASK) == _MB_START: * j += 1 # <<<<<<<<<<<<<< * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): - * ngram = bytes(utf8_word[i:j]) + * ngram = bytes(bytez[i:j]) */ __pyx_v_j = (__pyx_v_j + 1); } - /* "gensim/models/_utils_any2vec.pyx":153 - * while j < num_bytes and _is_utf8_continue(utf8_word[j]): + /* "gensim/models/_utils_any2vec.pyx":143 + * while j < num_bytes and (bytez[j] & _MB_MASK) == _MB_START: * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< - * ngram = bytes(utf8_word[i:j]) + * ngram = bytes(bytez[i:j]) * ngrams.append(ngram) */ __pyx_t_12 = ((__pyx_v_n >= __pyx_v_min_n) != 0); @@ -2967,68 +2669,61 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __pyx_L15_bool_binop_done:; if (__pyx_t_9) { - /* "gensim/models/_utils_any2vec.pyx":154 + /* "gensim/models/_utils_any2vec.pyx":144 * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): - * ngram = bytes(utf8_word[i:j]) # <<<<<<<<<<<<<< + * ngram = bytes(bytez[i:j]) # <<<<<<<<<<<<<< * ngrams.append(ngram) * n += 1 */ - if (unlikely(__pyx_v_utf8_word == Py_None)) { - PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); - __PYX_ERR(0, 154, __pyx_L1_error) - } - __pyx_t_1 = PySequence_GetSlice(__pyx_v_utf8_word, __pyx_v_i, __pyx_v_j); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 154, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(((const char*)__pyx_v_bytez) + __pyx_v_i, __pyx_v_j - __pyx_v_i); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 144, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 154, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 144, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_XDECREF_SET(__pyx_v_ngram, ((PyObject*)__pyx_t_3)); __pyx_t_3 = 0; - /* "gensim/models/_utils_any2vec.pyx":155 + /* "gensim/models/_utils_any2vec.pyx":145 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): - * ngram = bytes(utf8_word[i:j]) + * ngram = bytes(bytez[i:j]) * ngrams.append(ngram) # <<<<<<<<<<<<<< * n += 1 * return ngrams */ - __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_v_ngram); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 155, __pyx_L1_error) + __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_v_ngram); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 145, __pyx_L1_error) - /* "gensim/models/_utils_any2vec.pyx":153 - * while j < num_bytes and _is_utf8_continue(utf8_word[j]): + /* "gensim/models/_utils_any2vec.pyx":143 + * while j < num_bytes and (bytez[j] & _MB_MASK) == _MB_START: * j += 1 * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< - * ngram = bytes(utf8_word[i:j]) + * ngram = bytes(bytez[i:j]) * ngrams.append(ngram) */ } - /* "gensim/models/_utils_any2vec.pyx":156 - * ngram = bytes(utf8_word[i:j]) + /* "gensim/models/_utils_any2vec.pyx":146 + * ngram = bytes(bytez[i:j]) * ngrams.append(ngram) * n += 1 # <<<<<<<<<<<<<< * return ngrams - * */ __pyx_v_n = (__pyx_v_n + 1); } __pyx_L3_continue:; } - /* "gensim/models/_utils_any2vec.pyx":157 + /* "gensim/models/_utils_any2vec.pyx":147 * ngrams.append(ngram) * n += 1 * return ngrams # <<<<<<<<<<<<<< - * - * */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_ngrams); __pyx_r = __pyx_v_ngrams; goto __pyx_L0; - /* "gensim/models/_utils_any2vec.pyx":115 + /* "gensim/models/_utils_any2vec.pyx":104 * * * cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< @@ -3041,7 +2736,6 @@ static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(P __Pyx_XDECREF(__pyx_t_1); __Pyx_XDECREF(__pyx_t_2); __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_8); __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = 0; __pyx_L0:; @@ -3088,17 +2782,17 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes case 1: if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_min_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 1); __PYX_ERR(0, 115, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 1); __PYX_ERR(0, 104, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_n)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 2); __PYX_ERR(0, 115, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, 2); __PYX_ERR(0, 104, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams_bytes") < 0)) __PYX_ERR(0, 115, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams_bytes") < 0)) __PYX_ERR(0, 104, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { goto __pyx_L5_argtuple_error; @@ -3108,12 +2802,12 @@ static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes values[2] = PyTuple_GET_ITEM(__pyx_args, 2); } __pyx_v_word = values[0]; - __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 115, __pyx_L3_error) - __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 115, __pyx_L3_error) + __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 104, __pyx_L3_error) + __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 104, __pyx_L3_error) } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 115, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 104, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -3132,7 +2826,7 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("compute_ngrams_bytes", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 115, __pyx_L1_error) + __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 104, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -3149,1382 +2843,939 @@ static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes return __pyx_r; } -/* "gensim/models/_utils_any2vec.pyx":160 - * - * - * cpdef compute_ngrams_bytes_optimized(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< - * cdef unsigned char _mb_mask = 0xC0 - * cdef unsigned char _mb_start = 0x80 +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 + * # experimental exception made for __getbuffer__ and __releasebuffer__ + * # -- the details of this may change. + * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< + * # This implementation of getbuffer is geared towards Cython + * # requirements, and does not yet fulfill the PEP. */ -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_9compute_ngrams_bytes_optimized(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static PyObject *__pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes_optimized(PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n, CYTHON_UNUSED int __pyx_skip_dispatch) { - unsigned char __pyx_v__mb_mask; - unsigned char __pyx_v__mb_start; - PyObject *__pyx_v_utf8_word = 0; - unsigned char const *__pyx_v_bytez; - size_t __pyx_v_num_bytes; - size_t __pyx_v_j; - size_t __pyx_v_i; - size_t __pyx_v_n; - PyObject *__pyx_v_ngrams = NULL; - PyObject *__pyx_v_ngram = NULL; - PyObject *__pyx_r = NULL; +/* Python wrapper */ +static CYTHON_UNUSED int __pyx_pw_5numpy_7ndarray_1__getbuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +static CYTHON_UNUSED int __pyx_pw_5numpy_7ndarray_1__getbuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + int __pyx_r; __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannySetupContext("__getbuffer__ (wrapper)", 0); + __pyx_r = __pyx_pf_5numpy_7ndarray___getbuffer__(((PyArrayObject *)__pyx_v_self), ((Py_buffer *)__pyx_v_info), ((int)__pyx_v_flags)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + int __pyx_v_i; + int __pyx_v_ndim; + int __pyx_v_endian_detector; + int __pyx_v_little_endian; + int __pyx_v_t; + char *__pyx_v_f; + PyArray_Descr *__pyx_v_descr = 0; + int __pyx_v_offset; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; PyObject *__pyx_t_3 = NULL; - unsigned char const *__pyx_t_4; - Py_ssize_t __pyx_t_5; - size_t __pyx_t_6; - size_t __pyx_t_7; - size_t __pyx_t_8; - int __pyx_t_9; - size_t __pyx_t_10; - size_t __pyx_t_11; - int __pyx_t_12; - int __pyx_t_13; - int __pyx_t_14; - __Pyx_RefNannySetupContext("compute_ngrams_bytes_optimized", 0); + int __pyx_t_4; + int __pyx_t_5; + int __pyx_t_6; + PyArray_Descr *__pyx_t_7; + PyObject *__pyx_t_8 = NULL; + char *__pyx_t_9; + if (__pyx_v_info == NULL) { + PyErr_SetString(PyExc_BufferError, "PyObject_GetBuffer: view==NULL argument is obsolete"); + return -1; + } + __Pyx_RefNannySetupContext("__getbuffer__", 0); + __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(__pyx_v_info->obj); - /* "gensim/models/_utils_any2vec.pyx":161 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":265 + * + * cdef int i, ndim + * cdef int endian_detector = 1 # <<<<<<<<<<<<<< + * cdef bint little_endian = ((&endian_detector)[0] != 0) * - * cpdef compute_ngrams_bytes_optimized(word, unsigned int min_n, unsigned int max_n): - * cdef unsigned char _mb_mask = 0xC0 # <<<<<<<<<<<<<< - * cdef unsigned char _mb_start = 0x80 - * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") */ - __pyx_v__mb_mask = 0xC0; + __pyx_v_endian_detector = 1; - /* "gensim/models/_utils_any2vec.pyx":162 - * cpdef compute_ngrams_bytes_optimized(word, unsigned int min_n, unsigned int max_n): - * cdef unsigned char _mb_mask = 0xC0 - * cdef unsigned char _mb_start = 0x80 # <<<<<<<<<<<<<< - * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") - * cdef const unsigned char *bytez = utf8_word + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":266 + * cdef int i, ndim + * cdef int endian_detector = 1 + * cdef bint little_endian = ((&endian_detector)[0] != 0) # <<<<<<<<<<<<<< + * + * ndim = PyArray_NDIM(self) */ - __pyx_v__mb_start = 0x80; + __pyx_v_little_endian = ((((char *)(&__pyx_v_endian_detector))[0]) != 0); - /* "gensim/models/_utils_any2vec.pyx":163 - * cdef unsigned char _mb_mask = 0xC0 - * cdef unsigned char _mb_start = 0x80 - * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") # <<<<<<<<<<<<<< - * cdef const unsigned char *bytez = utf8_word - * cdef size_t num_bytes = len(utf8_word) + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":268 + * cdef bint little_endian = ((&endian_detector)[0] != 0) + * + * ndim = PyArray_NDIM(self) # <<<<<<<<<<<<<< + * + * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) */ - __pyx_t_2 = __Pyx_PyString_FormatSafe(__pyx_kp_s_s, __pyx_v_word); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 163, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 163, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = NULL; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { - __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); - if (likely(__pyx_t_2)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); - __Pyx_INCREF(__pyx_t_2); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_3, function); - } - } - __pyx_t_1 = (__pyx_t_2) ? __Pyx_PyObject_Call2Args(__pyx_t_3, __pyx_t_2, __pyx_kp_s_utf_8) : __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_kp_s_utf_8); - __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; - if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 163, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 163, __pyx_L1_error) - __pyx_v_utf8_word = ((PyObject*)__pyx_t_1); - __pyx_t_1 = 0; + __pyx_v_ndim = PyArray_NDIM(__pyx_v_self); - /* "gensim/models/_utils_any2vec.pyx":164 - * cdef unsigned char _mb_start = 0x80 - * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") - * cdef const unsigned char *bytez = utf8_word # <<<<<<<<<<<<<< - * cdef size_t num_bytes = len(utf8_word) - * cdef size_t j, i, n + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + * ndim = PyArray_NDIM(self) + * + * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< + * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): + * raise ValueError(u"ndarray is not C contiguous") */ - if (unlikely(__pyx_v_utf8_word == Py_None)) { - PyErr_SetString(PyExc_TypeError, "expected bytes, NoneType found"); - __PYX_ERR(0, 164, __pyx_L1_error) + __pyx_t_2 = (((__pyx_v_flags & PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS) != 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; } - __pyx_t_4 = __Pyx_PyBytes_AsUString(__pyx_v_utf8_word); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) __PYX_ERR(0, 164, __pyx_L1_error) - __pyx_v_bytez = __pyx_t_4; - /* "gensim/models/_utils_any2vec.pyx":165 - * cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") - * cdef const unsigned char *bytez = utf8_word - * cdef size_t num_bytes = len(utf8_word) # <<<<<<<<<<<<<< - * cdef size_t j, i, n + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":271 + * + * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) + * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): # <<<<<<<<<<<<<< + * raise ValueError(u"ndarray is not C contiguous") * */ - if (unlikely(__pyx_v_utf8_word == Py_None)) { - PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); - __PYX_ERR(0, 165, __pyx_L1_error) - } - __pyx_t_5 = PyBytes_GET_SIZE(__pyx_v_utf8_word); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 165, __pyx_L1_error) - __pyx_v_num_bytes = __pyx_t_5; + __pyx_t_2 = ((!(PyArray_CHKFLAGS(__pyx_v_self, NPY_ARRAY_C_CONTIGUOUS) != 0)) != 0); + __pyx_t_1 = __pyx_t_2; + __pyx_L4_bool_binop_done:; - /* "gensim/models/_utils_any2vec.pyx":168 - * cdef size_t j, i, n + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + * ndim = PyArray_NDIM(self) * - * ngrams = [] # <<<<<<<<<<<<<< - * for i in range(num_bytes): - * if bytez[i] & _mb_mask == _mb_start: + * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< + * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): + * raise ValueError(u"ndarray is not C contiguous") */ - __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 168, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_v_ngrams = ((PyObject*)__pyx_t_1); - __pyx_t_1 = 0; + if (unlikely(__pyx_t_1)) { - /* "gensim/models/_utils_any2vec.pyx":169 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 + * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) + * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): + * raise ValueError(u"ndarray is not C contiguous") # <<<<<<<<<<<<<< * - * ngrams = [] - * for i in range(num_bytes): # <<<<<<<<<<<<<< - * if bytez[i] & _mb_mask == _mb_start: - * continue + * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) */ - __pyx_t_6 = __pyx_v_num_bytes; - __pyx_t_7 = __pyx_t_6; - for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) { - __pyx_v_i = __pyx_t_8; + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 272, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(1, 272, __pyx_L1_error) - /* "gensim/models/_utils_any2vec.pyx":170 - * ngrams = [] - * for i in range(num_bytes): - * if bytez[i] & _mb_mask == _mb_start: # <<<<<<<<<<<<<< - * continue + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 + * ndim = PyArray_NDIM(self) * + * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< + * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): + * raise ValueError(u"ndarray is not C contiguous") */ - __pyx_t_9 = ((((__pyx_v_bytez[__pyx_v_i]) & __pyx_v__mb_mask) == __pyx_v__mb_start) != 0); - if (__pyx_t_9) { + } - /* "gensim/models/_utils_any2vec.pyx":171 - * for i in range(num_bytes): - * if bytez[i] & _mb_mask == _mb_start: - * continue # <<<<<<<<<<<<<< + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 + * raise ValueError(u"ndarray is not C contiguous") * - * j, n = i, 1 + * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< + * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): + * raise ValueError(u"ndarray is not Fortran contiguous") */ - goto __pyx_L3_continue; + __pyx_t_2 = (((__pyx_v_flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS) != 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L7_bool_binop_done; + } - /* "gensim/models/_utils_any2vec.pyx":170 - * ngrams = [] - * for i in range(num_bytes): - * if bytez[i] & _mb_mask == _mb_start: # <<<<<<<<<<<<<< - * continue + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":275 + * + * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) + * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): # <<<<<<<<<<<<<< + * raise ValueError(u"ndarray is not Fortran contiguous") * */ - } + __pyx_t_2 = ((!(PyArray_CHKFLAGS(__pyx_v_self, NPY_ARRAY_F_CONTIGUOUS) != 0)) != 0); + __pyx_t_1 = __pyx_t_2; + __pyx_L7_bool_binop_done:; - /* "gensim/models/_utils_any2vec.pyx":173 - * continue + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 + * raise ValueError(u"ndarray is not C contiguous") * - * j, n = i, 1 # <<<<<<<<<<<<<< - * while j < num_bytes and n <= max_n: - * j += 1 + * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< + * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): + * raise ValueError(u"ndarray is not Fortran contiguous") */ - __pyx_t_10 = __pyx_v_i; - __pyx_t_11 = 1; - __pyx_v_j = __pyx_t_10; - __pyx_v_n = __pyx_t_11; + if (unlikely(__pyx_t_1)) { - /* "gensim/models/_utils_any2vec.pyx":174 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 + * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) + * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): + * raise ValueError(u"ndarray is not Fortran contiguous") # <<<<<<<<<<<<<< * - * j, n = i, 1 - * while j < num_bytes and n <= max_n: # <<<<<<<<<<<<<< - * j += 1 - * while j < num_bytes and (bytez[j] & _mb_mask) == _mb_start: + * info.buf = PyArray_DATA(self) */ - while (1) { - __pyx_t_12 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); - if (__pyx_t_12) { - } else { - __pyx_t_9 = __pyx_t_12; - goto __pyx_L8_bool_binop_done; - } - __pyx_t_12 = ((__pyx_v_n <= __pyx_v_max_n) != 0); - __pyx_t_9 = __pyx_t_12; - __pyx_L8_bool_binop_done:; - if (!__pyx_t_9) break; + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 276, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(1, 276, __pyx_L1_error) - /* "gensim/models/_utils_any2vec.pyx":175 - * j, n = i, 1 - * while j < num_bytes and n <= max_n: - * j += 1 # <<<<<<<<<<<<<< - * while j < num_bytes and (bytez[j] & _mb_mask) == _mb_start: - * j += 1 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 + * raise ValueError(u"ndarray is not C contiguous") + * + * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< + * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): + * raise ValueError(u"ndarray is not Fortran contiguous") */ - __pyx_v_j = (__pyx_v_j + 1); + } - /* "gensim/models/_utils_any2vec.pyx":176 - * while j < num_bytes and n <= max_n: - * j += 1 - * while j < num_bytes and (bytez[j] & _mb_mask) == _mb_start: # <<<<<<<<<<<<<< - * j += 1 - * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":278 + * raise ValueError(u"ndarray is not Fortran contiguous") + * + * info.buf = PyArray_DATA(self) # <<<<<<<<<<<<<< + * info.ndim = ndim + * if sizeof(npy_intp) != sizeof(Py_ssize_t): */ - while (1) { - __pyx_t_12 = ((__pyx_v_j < __pyx_v_num_bytes) != 0); - if (__pyx_t_12) { - } else { - __pyx_t_9 = __pyx_t_12; - goto __pyx_L12_bool_binop_done; - } - __pyx_t_12 = ((((__pyx_v_bytez[__pyx_v_j]) & __pyx_v__mb_mask) == __pyx_v__mb_start) != 0); - __pyx_t_9 = __pyx_t_12; - __pyx_L12_bool_binop_done:; - if (!__pyx_t_9) break; + __pyx_v_info->buf = PyArray_DATA(__pyx_v_self); - /* "gensim/models/_utils_any2vec.pyx":177 - * j += 1 - * while j < num_bytes and (bytez[j] & _mb_mask) == _mb_start: - * j += 1 # <<<<<<<<<<<<<< - * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): - * ngram = bytes(bytez[i:j]) + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":279 + * + * info.buf = PyArray_DATA(self) + * info.ndim = ndim # <<<<<<<<<<<<<< + * if sizeof(npy_intp) != sizeof(Py_ssize_t): + * # Allocate new buffer for strides and shape info. */ - __pyx_v_j = (__pyx_v_j + 1); - } + __pyx_v_info->ndim = __pyx_v_ndim; - /* "gensim/models/_utils_any2vec.pyx":178 - * while j < num_bytes and (bytez[j] & _mb_mask) == _mb_start: - * j += 1 - * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< - * ngram = bytes(bytez[i:j]) - * ngrams.append(ngram) + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 + * info.buf = PyArray_DATA(self) + * info.ndim = ndim + * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< + * # Allocate new buffer for strides and shape info. + * # This is allocated as one block, strides first. */ - __pyx_t_12 = ((__pyx_v_n >= __pyx_v_min_n) != 0); - if (__pyx_t_12) { - } else { - __pyx_t_9 = __pyx_t_12; - goto __pyx_L15_bool_binop_done; - } - __pyx_t_13 = ((__pyx_v_n == 1) != 0); - if (__pyx_t_13) { - } else { - __pyx_t_12 = __pyx_t_13; - goto __pyx_L17_bool_binop_done; - } - __pyx_t_13 = ((__pyx_v_i == 0) != 0); - if (!__pyx_t_13) { - } else { - __pyx_t_12 = __pyx_t_13; - goto __pyx_L17_bool_binop_done; - } - __pyx_t_13 = ((__pyx_v_j == __pyx_v_num_bytes) != 0); - __pyx_t_12 = __pyx_t_13; - __pyx_L17_bool_binop_done:; - __pyx_t_13 = ((!__pyx_t_12) != 0); - __pyx_t_9 = __pyx_t_13; - __pyx_L15_bool_binop_done:; - if (__pyx_t_9) { + __pyx_t_1 = (((sizeof(npy_intp)) != (sizeof(Py_ssize_t))) != 0); + if (__pyx_t_1) { - /* "gensim/models/_utils_any2vec.pyx":179 - * j += 1 - * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): - * ngram = bytes(bytez[i:j]) # <<<<<<<<<<<<<< - * ngrams.append(ngram) - * n += 1 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":283 + * # Allocate new buffer for strides and shape info. + * # This is allocated as one block, strides first. + * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) # <<<<<<<<<<<<<< + * info.shape = info.strides + ndim + * for i in range(ndim): */ - __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(((const char*)__pyx_v_bytez) + __pyx_v_i, __pyx_v_j - __pyx_v_i); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 179, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_PyObject_CallOneArg(((PyObject *)(&PyBytes_Type)), __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 179, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_XDECREF_SET(__pyx_v_ngram, ((PyObject*)__pyx_t_3)); - __pyx_t_3 = 0; + __pyx_v_info->strides = ((Py_ssize_t *)PyObject_Malloc((((sizeof(Py_ssize_t)) * 2) * ((size_t)__pyx_v_ndim)))); - /* "gensim/models/_utils_any2vec.pyx":180 - * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): - * ngram = bytes(bytez[i:j]) - * ngrams.append(ngram) # <<<<<<<<<<<<<< - * n += 1 - * return ngrams + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":284 + * # This is allocated as one block, strides first. + * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) + * info.shape = info.strides + ndim # <<<<<<<<<<<<<< + * for i in range(ndim): + * info.strides[i] = PyArray_STRIDES(self)[i] */ - __pyx_t_14 = __Pyx_PyList_Append(__pyx_v_ngrams, __pyx_v_ngram); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 180, __pyx_L1_error) + __pyx_v_info->shape = (__pyx_v_info->strides + __pyx_v_ndim); - /* "gensim/models/_utils_any2vec.pyx":178 - * while j < num_bytes and (bytez[j] & _mb_mask) == _mb_start: - * j += 1 - * if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): # <<<<<<<<<<<<<< - * ngram = bytes(bytez[i:j]) - * ngrams.append(ngram) + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":285 + * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) + * info.shape = info.strides + ndim + * for i in range(ndim): # <<<<<<<<<<<<<< + * info.strides[i] = PyArray_STRIDES(self)[i] + * info.shape[i] = PyArray_DIMS(self)[i] */ - } + __pyx_t_4 = __pyx_v_ndim; + __pyx_t_5 = __pyx_t_4; + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_i = __pyx_t_6; - /* "gensim/models/_utils_any2vec.pyx":181 - * ngram = bytes(bytez[i:j]) - * ngrams.append(ngram) - * n += 1 # <<<<<<<<<<<<<< - * return ngrams + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":286 + * info.shape = info.strides + ndim + * for i in range(ndim): + * info.strides[i] = PyArray_STRIDES(self)[i] # <<<<<<<<<<<<<< + * info.shape[i] = PyArray_DIMS(self)[i] + * else: */ - __pyx_v_n = (__pyx_v_n + 1); - } - __pyx_L3_continue:; - } + (__pyx_v_info->strides[__pyx_v_i]) = (PyArray_STRIDES(__pyx_v_self)[__pyx_v_i]); - /* "gensim/models/_utils_any2vec.pyx":182 - * ngrams.append(ngram) - * n += 1 - * return ngrams # <<<<<<<<<<<<<< + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":287 + * for i in range(ndim): + * info.strides[i] = PyArray_STRIDES(self)[i] + * info.shape[i] = PyArray_DIMS(self)[i] # <<<<<<<<<<<<<< + * else: + * info.strides = PyArray_STRIDES(self) */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_ngrams); - __pyx_r = __pyx_v_ngrams; - goto __pyx_L0; + (__pyx_v_info->shape[__pyx_v_i]) = (PyArray_DIMS(__pyx_v_self)[__pyx_v_i]); + } - /* "gensim/models/_utils_any2vec.pyx":160 - * - * - * cpdef compute_ngrams_bytes_optimized(word, unsigned int min_n, unsigned int max_n): # <<<<<<<<<<<<<< - * cdef unsigned char _mb_mask = 0xC0 - * cdef unsigned char _mb_start = 0x80 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 + * info.buf = PyArray_DATA(self) + * info.ndim = ndim + * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< + * # Allocate new buffer for strides and shape info. + * # This is allocated as one block, strides first. */ + goto __pyx_L9; + } - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes_optimized", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_utf8_word); - __Pyx_XDECREF(__pyx_v_ngrams); - __Pyx_XDECREF(__pyx_v_ngram); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":289 + * info.shape[i] = PyArray_DIMS(self)[i] + * else: + * info.strides = PyArray_STRIDES(self) # <<<<<<<<<<<<<< + * info.shape = PyArray_DIMS(self) + * info.suboffsets = NULL + */ + /*else*/ { + __pyx_v_info->strides = ((Py_ssize_t *)PyArray_STRIDES(__pyx_v_self)); -/* Python wrapper */ -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_9compute_ngrams_bytes_optimized(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_6gensim_6models_14_utils_any2vec_8compute_ngrams_bytes_optimized[] = "compute_ngrams_bytes_optimized(word, unsigned int min_n, unsigned int max_n)"; -static PyObject *__pyx_pw_6gensim_6models_14_utils_any2vec_9compute_ngrams_bytes_optimized(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_word = 0; - unsigned int __pyx_v_min_n; - unsigned int __pyx_v_max_n; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("compute_ngrams_bytes_optimized (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_word,&__pyx_n_s_min_n,&__pyx_n_s_max_n,0}; - PyObject* values[3] = {0,0,0}; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - CYTHON_FALLTHROUGH; - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - CYTHON_FALLTHROUGH; - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - CYTHON_FALLTHROUGH; - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_word)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - CYTHON_FALLTHROUGH; - case 1: - if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_min_n)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes_optimized", 1, 3, 3, 1); __PYX_ERR(0, 160, __pyx_L3_error) - } - CYTHON_FALLTHROUGH; - case 2: - if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_n)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes_optimized", 1, 3, 3, 2); __PYX_ERR(0, 160, __pyx_L3_error) - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "compute_ngrams_bytes_optimized") < 0)) __PYX_ERR(0, 160, __pyx_L3_error) - } - } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { - goto __pyx_L5_argtuple_error; - } else { - values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - } - __pyx_v_word = values[0]; - __pyx_v_min_n = __Pyx_PyInt_As_unsigned_int(values[1]); if (unlikely((__pyx_v_min_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 160, __pyx_L3_error) - __pyx_v_max_n = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_max_n == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 160, __pyx_L3_error) + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":290 + * else: + * info.strides = PyArray_STRIDES(self) + * info.shape = PyArray_DIMS(self) # <<<<<<<<<<<<<< + * info.suboffsets = NULL + * info.itemsize = PyArray_ITEMSIZE(self) + */ + __pyx_v_info->shape = ((Py_ssize_t *)PyArray_DIMS(__pyx_v_self)); } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("compute_ngrams_bytes_optimized", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 160, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes_optimized", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_6gensim_6models_14_utils_any2vec_8compute_ngrams_bytes_optimized(__pyx_self, __pyx_v_word, __pyx_v_min_n, __pyx_v_max_n); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} + __pyx_L9:; -static PyObject *__pyx_pf_6gensim_6models_14_utils_any2vec_8compute_ngrams_bytes_optimized(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_word, unsigned int __pyx_v_min_n, unsigned int __pyx_v_max_n) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("compute_ngrams_bytes_optimized", 0); - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_6gensim_6models_14_utils_any2vec_compute_ngrams_bytes_optimized(__pyx_v_word, __pyx_v_min_n, __pyx_v_max_n, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 160, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":291 + * info.strides = PyArray_STRIDES(self) + * info.shape = PyArray_DIMS(self) + * info.suboffsets = NULL # <<<<<<<<<<<<<< + * info.itemsize = PyArray_ITEMSIZE(self) + * info.readonly = not PyArray_ISWRITEABLE(self) + */ + __pyx_v_info->suboffsets = NULL; - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("gensim.models._utils_any2vec.compute_ngrams_bytes_optimized", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":292 + * info.shape = PyArray_DIMS(self) + * info.suboffsets = NULL + * info.itemsize = PyArray_ITEMSIZE(self) # <<<<<<<<<<<<<< + * info.readonly = not PyArray_ISWRITEABLE(self) + * + */ + __pyx_v_info->itemsize = PyArray_ITEMSIZE(__pyx_v_self); -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 - * # experimental exception made for __getbuffer__ and __releasebuffer__ - * # -- the details of this may change. - * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< - * # This implementation of getbuffer is geared towards Cython - * # requirements, and does not yet fulfill the PEP. + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":293 + * info.suboffsets = NULL + * info.itemsize = PyArray_ITEMSIZE(self) + * info.readonly = not PyArray_ISWRITEABLE(self) # <<<<<<<<<<<<<< + * + * cdef int t */ + __pyx_v_info->readonly = (!(PyArray_ISWRITEABLE(__pyx_v_self) != 0)); -/* Python wrapper */ -static CYTHON_UNUSED int __pyx_pw_5numpy_7ndarray_1__getbuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ -static CYTHON_UNUSED int __pyx_pw_5numpy_7ndarray_1__getbuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__getbuffer__ (wrapper)", 0); - __pyx_r = __pyx_pf_5numpy_7ndarray___getbuffer__(((PyArrayObject *)__pyx_v_self), ((Py_buffer *)__pyx_v_info), ((int)__pyx_v_flags)); + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":296 + * + * cdef int t + * cdef char* f = NULL # <<<<<<<<<<<<<< + * cdef dtype descr = PyArray_DESCR(self) + * cdef int offset + */ + __pyx_v_f = NULL; - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":297 + * cdef int t + * cdef char* f = NULL + * cdef dtype descr = PyArray_DESCR(self) # <<<<<<<<<<<<<< + * cdef int offset + * + */ + __pyx_t_7 = PyArray_DESCR(__pyx_v_self); + __pyx_t_3 = ((PyObject *)__pyx_t_7); + __Pyx_INCREF(__pyx_t_3); + __pyx_v_descr = ((PyArray_Descr *)__pyx_t_3); + __pyx_t_3 = 0; -static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { - int __pyx_v_i; - int __pyx_v_ndim; - int __pyx_v_endian_detector; - int __pyx_v_little_endian; - int __pyx_v_t; - char *__pyx_v_f; - PyArray_Descr *__pyx_v_descr = 0; - int __pyx_v_offset; - int __pyx_r; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - int __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - int __pyx_t_4; - int __pyx_t_5; - int __pyx_t_6; - PyArray_Descr *__pyx_t_7; - PyObject *__pyx_t_8 = NULL; - char *__pyx_t_9; - if (__pyx_v_info == NULL) { - PyErr_SetString(PyExc_BufferError, "PyObject_GetBuffer: view==NULL argument is obsolete"); - return -1; - } - __Pyx_RefNannySetupContext("__getbuffer__", 0); - __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); - __Pyx_GIVEREF(__pyx_v_info->obj); - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":265 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":300 + * cdef int offset * - * cdef int i, ndim - * cdef int endian_detector = 1 # <<<<<<<<<<<<<< - * cdef bint little_endian = ((&endian_detector)[0] != 0) + * info.obj = self # <<<<<<<<<<<<<< * + * if not PyDataType_HASFIELDS(descr): */ - __pyx_v_endian_detector = 1; + __Pyx_INCREF(((PyObject *)__pyx_v_self)); + __Pyx_GIVEREF(((PyObject *)__pyx_v_self)); + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); + __pyx_v_info->obj = ((PyObject *)__pyx_v_self); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":266 - * cdef int i, ndim - * cdef int endian_detector = 1 - * cdef bint little_endian = ((&endian_detector)[0] != 0) # <<<<<<<<<<<<<< + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 + * info.obj = self * - * ndim = PyArray_NDIM(self) + * if not PyDataType_HASFIELDS(descr): # <<<<<<<<<<<<<< + * t = descr.type_num + * if ((descr.byteorder == c'>' and little_endian) or */ - __pyx_v_little_endian = ((((char *)(&__pyx_v_endian_detector))[0]) != 0); + __pyx_t_1 = ((!(PyDataType_HASFIELDS(__pyx_v_descr) != 0)) != 0); + if (__pyx_t_1) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":268 - * cdef bint little_endian = ((&endian_detector)[0] != 0) - * - * ndim = PyArray_NDIM(self) # <<<<<<<<<<<<<< + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":303 * - * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) + * if not PyDataType_HASFIELDS(descr): + * t = descr.type_num # <<<<<<<<<<<<<< + * if ((descr.byteorder == c'>' and little_endian) or + * (descr.byteorder == c'<' and not little_endian)): */ - __pyx_v_ndim = PyArray_NDIM(__pyx_v_self); + __pyx_t_4 = __pyx_v_descr->type_num; + __pyx_v_t = __pyx_t_4; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 - * ndim = PyArray_NDIM(self) - * - * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< - * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): - * raise ValueError(u"ndarray is not C contiguous") + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 + * if not PyDataType_HASFIELDS(descr): + * t = descr.type_num + * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< + * (descr.byteorder == c'<' and not little_endian)): + * raise ValueError(u"Non-native byte order not supported") */ - __pyx_t_2 = (((__pyx_v_flags & PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS) != 0); - if (__pyx_t_2) { - } else { - __pyx_t_1 = __pyx_t_2; - goto __pyx_L4_bool_binop_done; - } + __pyx_t_2 = ((__pyx_v_descr->byteorder == '>') != 0); + if (!__pyx_t_2) { + goto __pyx_L15_next_or; + } else { + } + __pyx_t_2 = (__pyx_v_little_endian != 0); + if (!__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L14_bool_binop_done; + } + __pyx_L15_next_or:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":271 - * - * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) - * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): # <<<<<<<<<<<<<< - * raise ValueError(u"ndarray is not C contiguous") - * + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":305 + * t = descr.type_num + * if ((descr.byteorder == c'>' and little_endian) or + * (descr.byteorder == c'<' and not little_endian)): # <<<<<<<<<<<<<< + * raise ValueError(u"Non-native byte order not supported") + * if t == NPY_BYTE: f = "b" */ - __pyx_t_2 = ((!(PyArray_CHKFLAGS(__pyx_v_self, NPY_ARRAY_C_CONTIGUOUS) != 0)) != 0); - __pyx_t_1 = __pyx_t_2; - __pyx_L4_bool_binop_done:; + __pyx_t_2 = ((__pyx_v_descr->byteorder == '<') != 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L14_bool_binop_done; + } + __pyx_t_2 = ((!(__pyx_v_little_endian != 0)) != 0); + __pyx_t_1 = __pyx_t_2; + __pyx_L14_bool_binop_done:; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 - * ndim = PyArray_NDIM(self) - * - * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< - * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): - * raise ValueError(u"ndarray is not C contiguous") + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 + * if not PyDataType_HASFIELDS(descr): + * t = descr.type_num + * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< + * (descr.byteorder == c'<' and not little_endian)): + * raise ValueError(u"Non-native byte order not supported") */ - if (unlikely(__pyx_t_1)) { + if (unlikely(__pyx_t_1)) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":272 - * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) - * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): - * raise ValueError(u"ndarray is not C contiguous") # <<<<<<<<<<<<<< - * - * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 + * if ((descr.byteorder == c'>' and little_endian) or + * (descr.byteorder == c'<' and not little_endian)): + * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< + * if t == NPY_BYTE: f = "b" + * elif t == NPY_UBYTE: f = "B" */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 272, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_Raise(__pyx_t_3, 0, 0, 0); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(1, 272, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 306, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(1, 306, __pyx_L1_error) - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":270 - * ndim = PyArray_NDIM(self) - * - * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< - * and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): - * raise ValueError(u"ndarray is not C contiguous") + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 + * if not PyDataType_HASFIELDS(descr): + * t = descr.type_num + * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< + * (descr.byteorder == c'<' and not little_endian)): + * raise ValueError(u"Non-native byte order not supported") */ - } + } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 - * raise ValueError(u"ndarray is not C contiguous") - * - * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< - * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): - * raise ValueError(u"ndarray is not Fortran contiguous") + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":307 + * (descr.byteorder == c'<' and not little_endian)): + * raise ValueError(u"Non-native byte order not supported") + * if t == NPY_BYTE: f = "b" # <<<<<<<<<<<<<< + * elif t == NPY_UBYTE: f = "B" + * elif t == NPY_SHORT: f = "h" */ - __pyx_t_2 = (((__pyx_v_flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS) != 0); - if (__pyx_t_2) { - } else { - __pyx_t_1 = __pyx_t_2; - goto __pyx_L7_bool_binop_done; - } + switch (__pyx_v_t) { + case NPY_BYTE: + __pyx_v_f = ((char *)"b"); + break; + case NPY_UBYTE: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":275 - * - * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) - * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): # <<<<<<<<<<<<<< - * raise ValueError(u"ndarray is not Fortran contiguous") - * + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":308 + * raise ValueError(u"Non-native byte order not supported") + * if t == NPY_BYTE: f = "b" + * elif t == NPY_UBYTE: f = "B" # <<<<<<<<<<<<<< + * elif t == NPY_SHORT: f = "h" + * elif t == NPY_USHORT: f = "H" */ - __pyx_t_2 = ((!(PyArray_CHKFLAGS(__pyx_v_self, NPY_ARRAY_F_CONTIGUOUS) != 0)) != 0); - __pyx_t_1 = __pyx_t_2; - __pyx_L7_bool_binop_done:; + __pyx_v_f = ((char *)"B"); + break; + case NPY_SHORT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 - * raise ValueError(u"ndarray is not C contiguous") - * - * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< - * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): - * raise ValueError(u"ndarray is not Fortran contiguous") + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":309 + * if t == NPY_BYTE: f = "b" + * elif t == NPY_UBYTE: f = "B" + * elif t == NPY_SHORT: f = "h" # <<<<<<<<<<<<<< + * elif t == NPY_USHORT: f = "H" + * elif t == NPY_INT: f = "i" */ - if (unlikely(__pyx_t_1)) { + __pyx_v_f = ((char *)"h"); + break; + case NPY_USHORT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":276 - * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) - * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): - * raise ValueError(u"ndarray is not Fortran contiguous") # <<<<<<<<<<<<<< - * - * info.buf = PyArray_DATA(self) + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":310 + * elif t == NPY_UBYTE: f = "B" + * elif t == NPY_SHORT: f = "h" + * elif t == NPY_USHORT: f = "H" # <<<<<<<<<<<<<< + * elif t == NPY_INT: f = "i" + * elif t == NPY_UINT: f = "I" */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 276, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_Raise(__pyx_t_3, 0, 0, 0); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(1, 276, __pyx_L1_error) + __pyx_v_f = ((char *)"H"); + break; + case NPY_INT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":274 - * raise ValueError(u"ndarray is not C contiguous") - * - * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< - * and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): - * raise ValueError(u"ndarray is not Fortran contiguous") - */ - } + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":311 + * elif t == NPY_SHORT: f = "h" + * elif t == NPY_USHORT: f = "H" + * elif t == NPY_INT: f = "i" # <<<<<<<<<<<<<< + * elif t == NPY_UINT: f = "I" + * elif t == NPY_LONG: f = "l" + */ + __pyx_v_f = ((char *)"i"); + break; + case NPY_UINT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":278 - * raise ValueError(u"ndarray is not Fortran contiguous") - * - * info.buf = PyArray_DATA(self) # <<<<<<<<<<<<<< - * info.ndim = ndim - * if sizeof(npy_intp) != sizeof(Py_ssize_t): + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":312 + * elif t == NPY_USHORT: f = "H" + * elif t == NPY_INT: f = "i" + * elif t == NPY_UINT: f = "I" # <<<<<<<<<<<<<< + * elif t == NPY_LONG: f = "l" + * elif t == NPY_ULONG: f = "L" */ - __pyx_v_info->buf = PyArray_DATA(__pyx_v_self); + __pyx_v_f = ((char *)"I"); + break; + case NPY_LONG: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":279 - * - * info.buf = PyArray_DATA(self) - * info.ndim = ndim # <<<<<<<<<<<<<< - * if sizeof(npy_intp) != sizeof(Py_ssize_t): - * # Allocate new buffer for strides and shape info. + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":313 + * elif t == NPY_INT: f = "i" + * elif t == NPY_UINT: f = "I" + * elif t == NPY_LONG: f = "l" # <<<<<<<<<<<<<< + * elif t == NPY_ULONG: f = "L" + * elif t == NPY_LONGLONG: f = "q" */ - __pyx_v_info->ndim = __pyx_v_ndim; + __pyx_v_f = ((char *)"l"); + break; + case NPY_ULONG: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 - * info.buf = PyArray_DATA(self) - * info.ndim = ndim - * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< - * # Allocate new buffer for strides and shape info. - * # This is allocated as one block, strides first. + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":314 + * elif t == NPY_UINT: f = "I" + * elif t == NPY_LONG: f = "l" + * elif t == NPY_ULONG: f = "L" # <<<<<<<<<<<<<< + * elif t == NPY_LONGLONG: f = "q" + * elif t == NPY_ULONGLONG: f = "Q" */ - __pyx_t_1 = (((sizeof(npy_intp)) != (sizeof(Py_ssize_t))) != 0); - if (__pyx_t_1) { + __pyx_v_f = ((char *)"L"); + break; + case NPY_LONGLONG: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":283 - * # Allocate new buffer for strides and shape info. - * # This is allocated as one block, strides first. - * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) # <<<<<<<<<<<<<< - * info.shape = info.strides + ndim - * for i in range(ndim): + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":315 + * elif t == NPY_LONG: f = "l" + * elif t == NPY_ULONG: f = "L" + * elif t == NPY_LONGLONG: f = "q" # <<<<<<<<<<<<<< + * elif t == NPY_ULONGLONG: f = "Q" + * elif t == NPY_FLOAT: f = "f" */ - __pyx_v_info->strides = ((Py_ssize_t *)PyObject_Malloc((((sizeof(Py_ssize_t)) * 2) * ((size_t)__pyx_v_ndim)))); + __pyx_v_f = ((char *)"q"); + break; + case NPY_ULONGLONG: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":284 - * # This is allocated as one block, strides first. - * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) - * info.shape = info.strides + ndim # <<<<<<<<<<<<<< - * for i in range(ndim): - * info.strides[i] = PyArray_STRIDES(self)[i] + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":316 + * elif t == NPY_ULONG: f = "L" + * elif t == NPY_LONGLONG: f = "q" + * elif t == NPY_ULONGLONG: f = "Q" # <<<<<<<<<<<<<< + * elif t == NPY_FLOAT: f = "f" + * elif t == NPY_DOUBLE: f = "d" */ - __pyx_v_info->shape = (__pyx_v_info->strides + __pyx_v_ndim); + __pyx_v_f = ((char *)"Q"); + break; + case NPY_FLOAT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":285 - * info.strides = PyObject_Malloc(sizeof(Py_ssize_t) * 2 * ndim) - * info.shape = info.strides + ndim - * for i in range(ndim): # <<<<<<<<<<<<<< - * info.strides[i] = PyArray_STRIDES(self)[i] - * info.shape[i] = PyArray_DIMS(self)[i] + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":317 + * elif t == NPY_LONGLONG: f = "q" + * elif t == NPY_ULONGLONG: f = "Q" + * elif t == NPY_FLOAT: f = "f" # <<<<<<<<<<<<<< + * elif t == NPY_DOUBLE: f = "d" + * elif t == NPY_LONGDOUBLE: f = "g" */ - __pyx_t_4 = __pyx_v_ndim; - __pyx_t_5 = __pyx_t_4; - for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { - __pyx_v_i = __pyx_t_6; + __pyx_v_f = ((char *)"f"); + break; + case NPY_DOUBLE: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":286 - * info.shape = info.strides + ndim - * for i in range(ndim): - * info.strides[i] = PyArray_STRIDES(self)[i] # <<<<<<<<<<<<<< - * info.shape[i] = PyArray_DIMS(self)[i] - * else: + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":318 + * elif t == NPY_ULONGLONG: f = "Q" + * elif t == NPY_FLOAT: f = "f" + * elif t == NPY_DOUBLE: f = "d" # <<<<<<<<<<<<<< + * elif t == NPY_LONGDOUBLE: f = "g" + * elif t == NPY_CFLOAT: f = "Zf" */ - (__pyx_v_info->strides[__pyx_v_i]) = (PyArray_STRIDES(__pyx_v_self)[__pyx_v_i]); + __pyx_v_f = ((char *)"d"); + break; + case NPY_LONGDOUBLE: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":287 - * for i in range(ndim): - * info.strides[i] = PyArray_STRIDES(self)[i] - * info.shape[i] = PyArray_DIMS(self)[i] # <<<<<<<<<<<<<< - * else: - * info.strides = PyArray_STRIDES(self) + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":319 + * elif t == NPY_FLOAT: f = "f" + * elif t == NPY_DOUBLE: f = "d" + * elif t == NPY_LONGDOUBLE: f = "g" # <<<<<<<<<<<<<< + * elif t == NPY_CFLOAT: f = "Zf" + * elif t == NPY_CDOUBLE: f = "Zd" */ - (__pyx_v_info->shape[__pyx_v_i]) = (PyArray_DIMS(__pyx_v_self)[__pyx_v_i]); - } + __pyx_v_f = ((char *)"g"); + break; + case NPY_CFLOAT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":280 - * info.buf = PyArray_DATA(self) - * info.ndim = ndim - * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< - * # Allocate new buffer for strides and shape info. - * # This is allocated as one block, strides first. + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":320 + * elif t == NPY_DOUBLE: f = "d" + * elif t == NPY_LONGDOUBLE: f = "g" + * elif t == NPY_CFLOAT: f = "Zf" # <<<<<<<<<<<<<< + * elif t == NPY_CDOUBLE: f = "Zd" + * elif t == NPY_CLONGDOUBLE: f = "Zg" */ - goto __pyx_L9; - } + __pyx_v_f = ((char *)"Zf"); + break; + case NPY_CDOUBLE: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":289 - * info.shape[i] = PyArray_DIMS(self)[i] - * else: - * info.strides = PyArray_STRIDES(self) # <<<<<<<<<<<<<< - * info.shape = PyArray_DIMS(self) - * info.suboffsets = NULL + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":321 + * elif t == NPY_LONGDOUBLE: f = "g" + * elif t == NPY_CFLOAT: f = "Zf" + * elif t == NPY_CDOUBLE: f = "Zd" # <<<<<<<<<<<<<< + * elif t == NPY_CLONGDOUBLE: f = "Zg" + * elif t == NPY_OBJECT: f = "O" */ - /*else*/ { - __pyx_v_info->strides = ((Py_ssize_t *)PyArray_STRIDES(__pyx_v_self)); + __pyx_v_f = ((char *)"Zd"); + break; + case NPY_CLONGDOUBLE: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":290 - * else: - * info.strides = PyArray_STRIDES(self) - * info.shape = PyArray_DIMS(self) # <<<<<<<<<<<<<< - * info.suboffsets = NULL - * info.itemsize = PyArray_ITEMSIZE(self) + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":322 + * elif t == NPY_CFLOAT: f = "Zf" + * elif t == NPY_CDOUBLE: f = "Zd" + * elif t == NPY_CLONGDOUBLE: f = "Zg" # <<<<<<<<<<<<<< + * elif t == NPY_OBJECT: f = "O" + * else: */ - __pyx_v_info->shape = ((Py_ssize_t *)PyArray_DIMS(__pyx_v_self)); - } - __pyx_L9:; + __pyx_v_f = ((char *)"Zg"); + break; + case NPY_OBJECT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":291 - * info.strides = PyArray_STRIDES(self) - * info.shape = PyArray_DIMS(self) - * info.suboffsets = NULL # <<<<<<<<<<<<<< - * info.itemsize = PyArray_ITEMSIZE(self) - * info.readonly = not PyArray_ISWRITEABLE(self) + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":323 + * elif t == NPY_CDOUBLE: f = "Zd" + * elif t == NPY_CLONGDOUBLE: f = "Zg" + * elif t == NPY_OBJECT: f = "O" # <<<<<<<<<<<<<< + * else: + * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) */ - __pyx_v_info->suboffsets = NULL; + __pyx_v_f = ((char *)"O"); + break; + default: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":292 - * info.shape = PyArray_DIMS(self) - * info.suboffsets = NULL - * info.itemsize = PyArray_ITEMSIZE(self) # <<<<<<<<<<<<<< - * info.readonly = not PyArray_ISWRITEABLE(self) - * + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":325 + * elif t == NPY_OBJECT: f = "O" + * else: + * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) # <<<<<<<<<<<<<< + * info.format = f + * return */ - __pyx_v_info->itemsize = PyArray_ITEMSIZE(__pyx_v_self); + __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_t); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 325, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_8 = PyUnicode_Format(__pyx_kp_u_unknown_dtype_code_in_numpy_pxd, __pyx_t_3); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 325, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_builtin_ValueError, __pyx_t_8); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 325, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(1, 325, __pyx_L1_error) + break; + } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":293 - * info.suboffsets = NULL - * info.itemsize = PyArray_ITEMSIZE(self) - * info.readonly = not PyArray_ISWRITEABLE(self) # <<<<<<<<<<<<<< - * - * cdef int t + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":326 + * else: + * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) + * info.format = f # <<<<<<<<<<<<<< + * return + * else: */ - __pyx_v_info->readonly = (!(PyArray_ISWRITEABLE(__pyx_v_self) != 0)); + __pyx_v_info->format = __pyx_v_f; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":296 - * - * cdef int t - * cdef char* f = NULL # <<<<<<<<<<<<<< - * cdef dtype descr = PyArray_DESCR(self) - * cdef int offset - */ - __pyx_v_f = NULL; - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":297 - * cdef int t - * cdef char* f = NULL - * cdef dtype descr = PyArray_DESCR(self) # <<<<<<<<<<<<<< - * cdef int offset - * - */ - __pyx_t_7 = PyArray_DESCR(__pyx_v_self); - __pyx_t_3 = ((PyObject *)__pyx_t_7); - __Pyx_INCREF(__pyx_t_3); - __pyx_v_descr = ((PyArray_Descr *)__pyx_t_3); - __pyx_t_3 = 0; - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":300 - * cdef int offset - * - * info.obj = self # <<<<<<<<<<<<<< - * - * if not PyDataType_HASFIELDS(descr): + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":327 + * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) + * info.format = f + * return # <<<<<<<<<<<<<< + * else: + * info.format = PyObject_Malloc(_buffer_format_string_len) */ - __Pyx_INCREF(((PyObject *)__pyx_v_self)); - __Pyx_GIVEREF(((PyObject *)__pyx_v_self)); - __Pyx_GOTREF(__pyx_v_info->obj); - __Pyx_DECREF(__pyx_v_info->obj); - __pyx_v_info->obj = ((PyObject *)__pyx_v_self); + __pyx_r = 0; + goto __pyx_L0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 * info.obj = self * * if not PyDataType_HASFIELDS(descr): # <<<<<<<<<<<<<< * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or */ - __pyx_t_1 = ((!(PyDataType_HASFIELDS(__pyx_v_descr) != 0)) != 0); - if (__pyx_t_1) { + } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":303 - * - * if not PyDataType_HASFIELDS(descr): - * t = descr.type_num # <<<<<<<<<<<<<< - * if ((descr.byteorder == c'>' and little_endian) or - * (descr.byteorder == c'<' and not little_endian)): + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":329 + * return + * else: + * info.format = PyObject_Malloc(_buffer_format_string_len) # <<<<<<<<<<<<<< + * info.format[0] = c'^' # Native data types, manual alignment + * offset = 0 */ - __pyx_t_4 = __pyx_v_descr->type_num; - __pyx_v_t = __pyx_t_4; + /*else*/ { + __pyx_v_info->format = ((char *)PyObject_Malloc(0xFF)); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 - * if not PyDataType_HASFIELDS(descr): - * t = descr.type_num - * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< - * (descr.byteorder == c'<' and not little_endian)): - * raise ValueError(u"Non-native byte order not supported") + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":330 + * else: + * info.format = PyObject_Malloc(_buffer_format_string_len) + * info.format[0] = c'^' # Native data types, manual alignment # <<<<<<<<<<<<<< + * offset = 0 + * f = _util_dtypestring(descr, info.format + 1, */ - __pyx_t_2 = ((__pyx_v_descr->byteorder == '>') != 0); - if (!__pyx_t_2) { - goto __pyx_L15_next_or; - } else { - } - __pyx_t_2 = (__pyx_v_little_endian != 0); - if (!__pyx_t_2) { - } else { - __pyx_t_1 = __pyx_t_2; - goto __pyx_L14_bool_binop_done; - } - __pyx_L15_next_or:; + (__pyx_v_info->format[0]) = '^'; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":305 - * t = descr.type_num - * if ((descr.byteorder == c'>' and little_endian) or - * (descr.byteorder == c'<' and not little_endian)): # <<<<<<<<<<<<<< - * raise ValueError(u"Non-native byte order not supported") - * if t == NPY_BYTE: f = "b" + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":331 + * info.format = PyObject_Malloc(_buffer_format_string_len) + * info.format[0] = c'^' # Native data types, manual alignment + * offset = 0 # <<<<<<<<<<<<<< + * f = _util_dtypestring(descr, info.format + 1, + * info.format + _buffer_format_string_len, */ - __pyx_t_2 = ((__pyx_v_descr->byteorder == '<') != 0); - if (__pyx_t_2) { - } else { - __pyx_t_1 = __pyx_t_2; - goto __pyx_L14_bool_binop_done; - } - __pyx_t_2 = ((!(__pyx_v_little_endian != 0)) != 0); - __pyx_t_1 = __pyx_t_2; - __pyx_L14_bool_binop_done:; + __pyx_v_offset = 0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 - * if not PyDataType_HASFIELDS(descr): - * t = descr.type_num - * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< - * (descr.byteorder == c'<' and not little_endian)): - * raise ValueError(u"Non-native byte order not supported") + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":332 + * info.format[0] = c'^' # Native data types, manual alignment + * offset = 0 + * f = _util_dtypestring(descr, info.format + 1, # <<<<<<<<<<<<<< + * info.format + _buffer_format_string_len, + * &offset) */ - if (unlikely(__pyx_t_1)) { + __pyx_t_9 = __pyx_f_5numpy__util_dtypestring(__pyx_v_descr, (__pyx_v_info->format + 1), (__pyx_v_info->format + 0xFF), (&__pyx_v_offset)); if (unlikely(__pyx_t_9 == ((char *)NULL))) __PYX_ERR(1, 332, __pyx_L1_error) + __pyx_v_f = __pyx_t_9; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":306 - * if ((descr.byteorder == c'>' and little_endian) or - * (descr.byteorder == c'<' and not little_endian)): - * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< - * if t == NPY_BYTE: f = "b" - * elif t == NPY_UBYTE: f = "B" + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":335 + * info.format + _buffer_format_string_len, + * &offset) + * f[0] = c'\0' # Terminate format string # <<<<<<<<<<<<<< + * + * def __releasebuffer__(ndarray self, Py_buffer* info): */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 306, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_Raise(__pyx_t_3, 0, 0, 0); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(1, 306, __pyx_L1_error) + (__pyx_v_f[0]) = '\x00'; + } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":304 - * if not PyDataType_HASFIELDS(descr): - * t = descr.type_num - * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< - * (descr.byteorder == c'<' and not little_endian)): - * raise ValueError(u"Non-native byte order not supported") + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 + * # experimental exception made for __getbuffer__ and __releasebuffer__ + * # -- the details of this may change. + * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< + * # This implementation of getbuffer is geared towards Cython + * # requirements, and does not yet fulfill the PEP. */ - } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":307 - * (descr.byteorder == c'<' and not little_endian)): - * raise ValueError(u"Non-native byte order not supported") - * if t == NPY_BYTE: f = "b" # <<<<<<<<<<<<<< - * elif t == NPY_UBYTE: f = "B" - * elif t == NPY_SHORT: f = "h" - */ - switch (__pyx_v_t) { - case NPY_BYTE: - __pyx_v_f = ((char *)"b"); - break; - case NPY_UBYTE: + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("numpy.ndarray.__getbuffer__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + if (__pyx_v_info->obj != NULL) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + goto __pyx_L2; + __pyx_L0:; + if (__pyx_v_info->obj == Py_None) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + __pyx_L2:; + __Pyx_XDECREF((PyObject *)__pyx_v_descr); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":308 - * raise ValueError(u"Non-native byte order not supported") - * if t == NPY_BYTE: f = "b" - * elif t == NPY_UBYTE: f = "B" # <<<<<<<<<<<<<< - * elif t == NPY_SHORT: f = "h" - * elif t == NPY_USHORT: f = "H" +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 + * f[0] = c'\0' # Terminate format string + * + * def __releasebuffer__(ndarray self, Py_buffer* info): # <<<<<<<<<<<<<< + * if PyArray_HASFIELDS(self): + * PyObject_Free(info.format) */ - __pyx_v_f = ((char *)"B"); - break; - case NPY_SHORT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":309 - * if t == NPY_BYTE: f = "b" - * elif t == NPY_UBYTE: f = "B" - * elif t == NPY_SHORT: f = "h" # <<<<<<<<<<<<<< - * elif t == NPY_USHORT: f = "H" - * elif t == NPY_INT: f = "i" - */ - __pyx_v_f = ((char *)"h"); - break; - case NPY_USHORT: +/* Python wrapper */ +static CYTHON_UNUSED void __pyx_pw_5numpy_7ndarray_3__releasebuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info); /*proto*/ +static CYTHON_UNUSED void __pyx_pw_5numpy_7ndarray_3__releasebuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__releasebuffer__ (wrapper)", 0); + __pyx_pf_5numpy_7ndarray_2__releasebuffer__(((PyArrayObject *)__pyx_v_self), ((Py_buffer *)__pyx_v_info)); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":310 - * elif t == NPY_UBYTE: f = "B" - * elif t == NPY_SHORT: f = "h" - * elif t == NPY_USHORT: f = "H" # <<<<<<<<<<<<<< - * elif t == NPY_INT: f = "i" - * elif t == NPY_UINT: f = "I" - */ - __pyx_v_f = ((char *)"H"); - break; - case NPY_INT: + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":311 - * elif t == NPY_SHORT: f = "h" - * elif t == NPY_USHORT: f = "H" - * elif t == NPY_INT: f = "i" # <<<<<<<<<<<<<< - * elif t == NPY_UINT: f = "I" - * elif t == NPY_LONG: f = "l" +static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info) { + __Pyx_RefNannyDeclarations + int __pyx_t_1; + __Pyx_RefNannySetupContext("__releasebuffer__", 0); + + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 + * + * def __releasebuffer__(ndarray self, Py_buffer* info): + * if PyArray_HASFIELDS(self): # <<<<<<<<<<<<<< + * PyObject_Free(info.format) + * if sizeof(npy_intp) != sizeof(Py_ssize_t): */ - __pyx_v_f = ((char *)"i"); - break; - case NPY_UINT: + __pyx_t_1 = (PyArray_HASFIELDS(__pyx_v_self) != 0); + if (__pyx_t_1) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":312 - * elif t == NPY_USHORT: f = "H" - * elif t == NPY_INT: f = "i" - * elif t == NPY_UINT: f = "I" # <<<<<<<<<<<<<< - * elif t == NPY_LONG: f = "l" - * elif t == NPY_ULONG: f = "L" + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":339 + * def __releasebuffer__(ndarray self, Py_buffer* info): + * if PyArray_HASFIELDS(self): + * PyObject_Free(info.format) # <<<<<<<<<<<<<< + * if sizeof(npy_intp) != sizeof(Py_ssize_t): + * PyObject_Free(info.strides) */ - __pyx_v_f = ((char *)"I"); - break; - case NPY_LONG: + PyObject_Free(__pyx_v_info->format); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":313 - * elif t == NPY_INT: f = "i" - * elif t == NPY_UINT: f = "I" - * elif t == NPY_LONG: f = "l" # <<<<<<<<<<<<<< - * elif t == NPY_ULONG: f = "L" - * elif t == NPY_LONGLONG: f = "q" + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 + * + * def __releasebuffer__(ndarray self, Py_buffer* info): + * if PyArray_HASFIELDS(self): # <<<<<<<<<<<<<< + * PyObject_Free(info.format) + * if sizeof(npy_intp) != sizeof(Py_ssize_t): */ - __pyx_v_f = ((char *)"l"); - break; - case NPY_ULONG: + } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":314 - * elif t == NPY_UINT: f = "I" - * elif t == NPY_LONG: f = "l" - * elif t == NPY_ULONG: f = "L" # <<<<<<<<<<<<<< - * elif t == NPY_LONGLONG: f = "q" - * elif t == NPY_ULONGLONG: f = "Q" + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 + * if PyArray_HASFIELDS(self): + * PyObject_Free(info.format) + * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< + * PyObject_Free(info.strides) + * # info.shape was stored after info.strides in the same block */ - __pyx_v_f = ((char *)"L"); - break; - case NPY_LONGLONG: + __pyx_t_1 = (((sizeof(npy_intp)) != (sizeof(Py_ssize_t))) != 0); + if (__pyx_t_1) { - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":315 - * elif t == NPY_LONG: f = "l" - * elif t == NPY_ULONG: f = "L" - * elif t == NPY_LONGLONG: f = "q" # <<<<<<<<<<<<<< - * elif t == NPY_ULONGLONG: f = "Q" - * elif t == NPY_FLOAT: f = "f" + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":341 + * PyObject_Free(info.format) + * if sizeof(npy_intp) != sizeof(Py_ssize_t): + * PyObject_Free(info.strides) # <<<<<<<<<<<<<< + * # info.shape was stored after info.strides in the same block + * */ - __pyx_v_f = ((char *)"q"); - break; - case NPY_ULONGLONG: + PyObject_Free(__pyx_v_info->strides); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":316 - * elif t == NPY_ULONG: f = "L" - * elif t == NPY_LONGLONG: f = "q" - * elif t == NPY_ULONGLONG: f = "Q" # <<<<<<<<<<<<<< - * elif t == NPY_FLOAT: f = "f" - * elif t == NPY_DOUBLE: f = "d" + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 + * if PyArray_HASFIELDS(self): + * PyObject_Free(info.format) + * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< + * PyObject_Free(info.strides) + * # info.shape was stored after info.strides in the same block */ - __pyx_v_f = ((char *)"Q"); - break; - case NPY_FLOAT: + } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":317 - * elif t == NPY_LONGLONG: f = "q" - * elif t == NPY_ULONGLONG: f = "Q" - * elif t == NPY_FLOAT: f = "f" # <<<<<<<<<<<<<< - * elif t == NPY_DOUBLE: f = "d" - * elif t == NPY_LONGDOUBLE: f = "g" + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 + * f[0] = c'\0' # Terminate format string + * + * def __releasebuffer__(ndarray self, Py_buffer* info): # <<<<<<<<<<<<<< + * if PyArray_HASFIELDS(self): + * PyObject_Free(info.format) */ - __pyx_v_f = ((char *)"f"); - break; - case NPY_DOUBLE: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":318 - * elif t == NPY_ULONGLONG: f = "Q" - * elif t == NPY_FLOAT: f = "f" - * elif t == NPY_DOUBLE: f = "d" # <<<<<<<<<<<<<< - * elif t == NPY_LONGDOUBLE: f = "g" - * elif t == NPY_CFLOAT: f = "Zf" - */ - __pyx_v_f = ((char *)"d"); - break; - case NPY_LONGDOUBLE: + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":319 - * elif t == NPY_FLOAT: f = "f" - * elif t == NPY_DOUBLE: f = "d" - * elif t == NPY_LONGDOUBLE: f = "g" # <<<<<<<<<<<<<< - * elif t == NPY_CFLOAT: f = "Zf" - * elif t == NPY_CDOUBLE: f = "Zd" +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 + * ctypedef npy_cdouble complex_t + * + * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(1, a) + * */ - __pyx_v_f = ((char *)"g"); - break; - case NPY_CFLOAT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":320 - * elif t == NPY_DOUBLE: f = "d" - * elif t == NPY_LONGDOUBLE: f = "g" - * elif t == NPY_CFLOAT: f = "Zf" # <<<<<<<<<<<<<< - * elif t == NPY_CDOUBLE: f = "Zd" - * elif t == NPY_CLONGDOUBLE: f = "Zg" - */ - __pyx_v_f = ((char *)"Zf"); - break; - case NPY_CDOUBLE: +static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__pyx_v_a) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("PyArray_MultiIterNew1", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":321 - * elif t == NPY_LONGDOUBLE: f = "g" - * elif t == NPY_CFLOAT: f = "Zf" - * elif t == NPY_CDOUBLE: f = "Zd" # <<<<<<<<<<<<<< - * elif t == NPY_CLONGDOUBLE: f = "Zg" - * elif t == NPY_OBJECT: f = "O" + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":822 + * + * cdef inline object PyArray_MultiIterNew1(a): + * return PyArray_MultiIterNew(1, a) # <<<<<<<<<<<<<< + * + * cdef inline object PyArray_MultiIterNew2(a, b): */ - __pyx_v_f = ((char *)"Zd"); - break; - case NPY_CLONGDOUBLE: + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyArray_MultiIterNew(1, ((void *)__pyx_v_a)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 822, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":322 - * elif t == NPY_CFLOAT: f = "Zf" - * elif t == NPY_CDOUBLE: f = "Zd" - * elif t == NPY_CLONGDOUBLE: f = "Zg" # <<<<<<<<<<<<<< - * elif t == NPY_OBJECT: f = "O" - * else: + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 + * ctypedef npy_cdouble complex_t + * + * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(1, a) + * */ - __pyx_v_f = ((char *)"Zg"); - break; - case NPY_OBJECT: - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":323 - * elif t == NPY_CDOUBLE: f = "Zd" - * elif t == NPY_CLONGDOUBLE: f = "Zg" - * elif t == NPY_OBJECT: f = "O" # <<<<<<<<<<<<<< - * else: - * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) - */ - __pyx_v_f = ((char *)"O"); - break; - default: + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("numpy.PyArray_MultiIterNew1", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":325 - * elif t == NPY_OBJECT: f = "O" - * else: - * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) # <<<<<<<<<<<<<< - * info.format = f - * return +/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 + * return PyArray_MultiIterNew(1, a) + * + * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(2, a, b) + * */ - __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_t); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 325, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_8 = PyUnicode_Format(__pyx_kp_u_unknown_dtype_code_in_numpy_pxd, __pyx_t_3); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 325, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_builtin_ValueError, __pyx_t_8); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 325, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_Raise(__pyx_t_3, 0, 0, 0); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(1, 325, __pyx_L1_error) - break; - } - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":326 - * else: - * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) - * info.format = f # <<<<<<<<<<<<<< - * return - * else: - */ - __pyx_v_info->format = __pyx_v_f; +static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__pyx_v_a, PyObject *__pyx_v_b) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("PyArray_MultiIterNew2", 0); - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":327 - * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) - * info.format = f - * return # <<<<<<<<<<<<<< - * else: - * info.format = PyObject_Malloc(_buffer_format_string_len) + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":825 + * + * cdef inline object PyArray_MultiIterNew2(a, b): + * return PyArray_MultiIterNew(2, a, b) # <<<<<<<<<<<<<< + * + * cdef inline object PyArray_MultiIterNew3(a, b, c): */ - __pyx_r = 0; - goto __pyx_L0; + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyArray_MultiIterNew(2, ((void *)__pyx_v_a), ((void *)__pyx_v_b)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 825, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":302 - * info.obj = self - * - * if not PyDataType_HASFIELDS(descr): # <<<<<<<<<<<<<< - * t = descr.type_num - * if ((descr.byteorder == c'>' and little_endian) or - */ - } - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":329 - * return - * else: - * info.format = PyObject_Malloc(_buffer_format_string_len) # <<<<<<<<<<<<<< - * info.format[0] = c'^' # Native data types, manual alignment - * offset = 0 - */ - /*else*/ { - __pyx_v_info->format = ((char *)PyObject_Malloc(0xFF)); - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":330 - * else: - * info.format = PyObject_Malloc(_buffer_format_string_len) - * info.format[0] = c'^' # Native data types, manual alignment # <<<<<<<<<<<<<< - * offset = 0 - * f = _util_dtypestring(descr, info.format + 1, - */ - (__pyx_v_info->format[0]) = '^'; - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":331 - * info.format = PyObject_Malloc(_buffer_format_string_len) - * info.format[0] = c'^' # Native data types, manual alignment - * offset = 0 # <<<<<<<<<<<<<< - * f = _util_dtypestring(descr, info.format + 1, - * info.format + _buffer_format_string_len, - */ - __pyx_v_offset = 0; - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":332 - * info.format[0] = c'^' # Native data types, manual alignment - * offset = 0 - * f = _util_dtypestring(descr, info.format + 1, # <<<<<<<<<<<<<< - * info.format + _buffer_format_string_len, - * &offset) - */ - __pyx_t_9 = __pyx_f_5numpy__util_dtypestring(__pyx_v_descr, (__pyx_v_info->format + 1), (__pyx_v_info->format + 0xFF), (&__pyx_v_offset)); if (unlikely(__pyx_t_9 == ((char *)NULL))) __PYX_ERR(1, 332, __pyx_L1_error) - __pyx_v_f = __pyx_t_9; - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":335 - * info.format + _buffer_format_string_len, - * &offset) - * f[0] = c'\0' # Terminate format string # <<<<<<<<<<<<<< - * - * def __releasebuffer__(ndarray self, Py_buffer* info): - */ - (__pyx_v_f[0]) = '\x00'; - } - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":258 - * # experimental exception made for __getbuffer__ and __releasebuffer__ - * # -- the details of this may change. - * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< - * # This implementation of getbuffer is geared towards Cython - * # requirements, and does not yet fulfill the PEP. - */ - - /* function exit code */ - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_8); - __Pyx_AddTraceback("numpy.ndarray.__getbuffer__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - if (__pyx_v_info->obj != NULL) { - __Pyx_GOTREF(__pyx_v_info->obj); - __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; - } - goto __pyx_L2; - __pyx_L0:; - if (__pyx_v_info->obj == Py_None) { - __Pyx_GOTREF(__pyx_v_info->obj); - __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; - } - __pyx_L2:; - __Pyx_XDECREF((PyObject *)__pyx_v_descr); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 - * f[0] = c'\0' # Terminate format string - * - * def __releasebuffer__(ndarray self, Py_buffer* info): # <<<<<<<<<<<<<< - * if PyArray_HASFIELDS(self): - * PyObject_Free(info.format) - */ - -/* Python wrapper */ -static CYTHON_UNUSED void __pyx_pw_5numpy_7ndarray_3__releasebuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info); /*proto*/ -static CYTHON_UNUSED void __pyx_pw_5numpy_7ndarray_3__releasebuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__releasebuffer__ (wrapper)", 0); - __pyx_pf_5numpy_7ndarray_2__releasebuffer__(((PyArrayObject *)__pyx_v_self), ((Py_buffer *)__pyx_v_info)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); -} - -static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info) { - __Pyx_RefNannyDeclarations - int __pyx_t_1; - __Pyx_RefNannySetupContext("__releasebuffer__", 0); - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 - * - * def __releasebuffer__(ndarray self, Py_buffer* info): - * if PyArray_HASFIELDS(self): # <<<<<<<<<<<<<< - * PyObject_Free(info.format) - * if sizeof(npy_intp) != sizeof(Py_ssize_t): - */ - __pyx_t_1 = (PyArray_HASFIELDS(__pyx_v_self) != 0); - if (__pyx_t_1) { - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":339 - * def __releasebuffer__(ndarray self, Py_buffer* info): - * if PyArray_HASFIELDS(self): - * PyObject_Free(info.format) # <<<<<<<<<<<<<< - * if sizeof(npy_intp) != sizeof(Py_ssize_t): - * PyObject_Free(info.strides) - */ - PyObject_Free(__pyx_v_info->format); - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":338 - * - * def __releasebuffer__(ndarray self, Py_buffer* info): - * if PyArray_HASFIELDS(self): # <<<<<<<<<<<<<< - * PyObject_Free(info.format) - * if sizeof(npy_intp) != sizeof(Py_ssize_t): - */ - } - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 - * if PyArray_HASFIELDS(self): - * PyObject_Free(info.format) - * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< - * PyObject_Free(info.strides) - * # info.shape was stored after info.strides in the same block - */ - __pyx_t_1 = (((sizeof(npy_intp)) != (sizeof(Py_ssize_t))) != 0); - if (__pyx_t_1) { - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":341 - * PyObject_Free(info.format) - * if sizeof(npy_intp) != sizeof(Py_ssize_t): - * PyObject_Free(info.strides) # <<<<<<<<<<<<<< - * # info.shape was stored after info.strides in the same block - * - */ - PyObject_Free(__pyx_v_info->strides); - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":340 - * if PyArray_HASFIELDS(self): - * PyObject_Free(info.format) - * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< - * PyObject_Free(info.strides) - * # info.shape was stored after info.strides in the same block - */ - } - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":337 - * f[0] = c'\0' # Terminate format string - * - * def __releasebuffer__(ndarray self, Py_buffer* info): # <<<<<<<<<<<<<< - * if PyArray_HASFIELDS(self): - * PyObject_Free(info.format) - */ - - /* function exit code */ - __Pyx_RefNannyFinishContext(); -} - -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 - * ctypedef npy_cdouble complex_t - * - * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< - * return PyArray_MultiIterNew(1, a) - * - */ - -static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__pyx_v_a) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("PyArray_MultiIterNew1", 0); - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":822 - * - * cdef inline object PyArray_MultiIterNew1(a): - * return PyArray_MultiIterNew(1, a) # <<<<<<<<<<<<<< - * - * cdef inline object PyArray_MultiIterNew2(a, b): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyArray_MultiIterNew(1, ((void *)__pyx_v_a)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 822, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":821 - * ctypedef npy_cdouble complex_t - * - * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< - * return PyArray_MultiIterNew(1, a) - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("numpy.PyArray_MultiIterNew1", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 - * return PyArray_MultiIterNew(1, a) - * - * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< - * return PyArray_MultiIterNew(2, a, b) - * - */ - -static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__pyx_v_a, PyObject *__pyx_v_b) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("PyArray_MultiIterNew2", 0); - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":825 - * - * cdef inline object PyArray_MultiIterNew2(a, b): - * return PyArray_MultiIterNew(2, a, b) # <<<<<<<<<<<<<< - * - * cdef inline object PyArray_MultiIterNew3(a, b, c): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyArray_MultiIterNew(2, ((void *)__pyx_v_a), ((void *)__pyx_v_b)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 825, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 - * return PyArray_MultiIterNew(1, a) + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":824 + * return PyArray_MultiIterNew(1, a) * * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< * return PyArray_MultiIterNew(2, a, b) @@ -6010,230 +5261,11 @@ static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { return __pyx_r; } -/* "cfunc.to_py":65 - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): - * def wrap(object b): # <<<<<<<<<<<<<< - * """wrap(b)""" - * return f(b) - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap(PyObject *__pyx_self, PyObject *__pyx_v_b); /*proto*/ -static char __pyx_doc_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap[] = "wrap(b)"; -static PyMethodDef __pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap = {"wrap", (PyCFunction)__pyx_pw_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, METH_O, __pyx_doc_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap}; -static PyObject *__pyx_pw_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap(PyObject *__pyx_self, PyObject *__pyx_v_b) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("wrap (wrapper)", 0); - __pyx_r = __pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(__pyx_self, ((PyObject *)__pyx_v_b)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(PyObject *__pyx_self, PyObject *__pyx_v_b) { - struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_cur_scope; - struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_outer_scope; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("wrap", 0); - __pyx_outer_scope = (struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *) __Pyx_CyFunction_GetClosure(__pyx_self); - __pyx_cur_scope = __pyx_outer_scope; - - /* "cfunc.to_py":67 - * def wrap(object b): - * """wrap(b)""" - * return f(b) # <<<<<<<<<<<<<< - * return wrap - * - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_cur_scope->__pyx_v_f(__pyx_v_b); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 67, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "cfunc.to_py":65 - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): - * def wrap(object b): # <<<<<<<<<<<<<< - * """wrap(b)""" - * return f(b) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("cfunc.to_py.__Pyx_CFunc_object____object___to_py.wrap", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "cfunc.to_py":64 - * - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< - * def wrap(object b): - * """wrap(b)""" - */ - -static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*__pyx_v_f)(PyObject *)) { - struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_cur_scope; - PyObject *__pyx_v_wrap = 0; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__Pyx_CFunc_object____object___to_py", 0); - __pyx_cur_scope = (struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(__pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py, __pyx_empty_tuple, NULL); - if (unlikely(!__pyx_cur_scope)) { - __pyx_cur_scope = ((struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)Py_None); - __Pyx_INCREF(Py_None); - __PYX_ERR(2, 64, __pyx_L1_error) - } else { - __Pyx_GOTREF(__pyx_cur_scope); - } - __pyx_cur_scope->__pyx_v_f = __pyx_v_f; - - /* "cfunc.to_py":65 - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): - * def wrap(object b): # <<<<<<<<<<<<<< - * """wrap(b)""" - * return f(b) - */ - __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, 0, __pyx_n_s_Pyx_CFunc_object____object___t, ((PyObject*)__pyx_cur_scope), __pyx_n_s_cfunc_to_py, __pyx_d, ((PyObject *)__pyx_codeobj__11)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 65, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_v_wrap = __pyx_t_1; - __pyx_t_1 = 0; - - /* "cfunc.to_py":68 - * """wrap(b)""" - * return f(b) - * return wrap # <<<<<<<<<<<<<< - * - * - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_wrap); - __pyx_r = __pyx_v_wrap; - goto __pyx_L0; - - /* "cfunc.to_py":64 - * - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< - * def wrap(object b): - * """wrap(b)""" - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("cfunc.to_py.__Pyx_CFunc_object____object___to_py", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_wrap); - __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[8]; -static int __pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py = 0; - -static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - PyObject *o; - if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)))) { - o = (PyObject*)__pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[--__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py]; - memset(o, 0, sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)); - (void) PyObject_INIT(o, t); - } else { - o = (*t->tp_alloc)(t, 0); - if (unlikely(!o)) return 0; - } - return o; -} - -static void __pyx_tp_dealloc___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyObject *o) { - if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)))) { - __pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py++] = ((struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)o); - } else { - (*Py_TYPE(o)->tp_free)(o); - } -} - -static PyTypeObject __pyx_scope_struct____Pyx_CFunc_object____object___to_py = { - PyVarObject_HEAD_INIT(0, 0) - "gensim.models._utils_any2vec.__pyx_scope_struct____Pyx_CFunc_object____object___to_py", /*tp_name*/ - sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc___pyx_scope_struct____Pyx_CFunc_object____object___to_py, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - static PyMethodDef __pyx_methods[] = { {"ft_hash_bytes", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_1ft_hash_bytes, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_ft_hash_bytes}, {"ft_hash_broken", (PyCFunction)__pyx_pw_6gensim_6models_14_utils_any2vec_3ft_hash_broken, METH_O, __pyx_doc_6gensim_6models_14_utils_any2vec_2ft_hash_broken}, {"compute_ngrams", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_6gensim_6models_14_utils_any2vec_5compute_ngrams, METH_VARARGS|METH_KEYWORDS, __pyx_doc_6gensim_6models_14_utils_any2vec_4compute_ngrams}, {"compute_ngrams_bytes", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_6gensim_6models_14_utils_any2vec_7compute_ngrams_bytes, METH_VARARGS|METH_KEYWORDS, __pyx_doc_6gensim_6models_14_utils_any2vec_6compute_ngrams_bytes}, - {"compute_ngrams_bytes_optimized", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_6gensim_6models_14_utils_any2vec_9compute_ngrams_bytes_optimized, METH_VARARGS|METH_KEYWORDS, __pyx_doc_6gensim_6models_14_utils_any2vec_8compute_ngrams_bytes_optimized}, {0, 0, 0, 0} }; @@ -6283,20 +5315,14 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_u_Format_string_allocated_too_shor, __pyx_k_Format_string_allocated_too_shor, sizeof(__pyx_k_Format_string_allocated_too_shor), 0, 1, 0, 0}, {&__pyx_kp_u_Format_string_allocated_too_shor_2, __pyx_k_Format_string_allocated_too_shor_2, sizeof(__pyx_k_Format_string_allocated_too_shor_2), 0, 1, 0, 0}, {&__pyx_n_s_ImportError, __pyx_k_ImportError, sizeof(__pyx_k_ImportError), 0, 0, 1, 1}, - {&__pyx_n_s_MB_MASK, __pyx_k_MB_MASK, sizeof(__pyx_k_MB_MASK), 0, 0, 1, 1}, - {&__pyx_n_s_MB_START, __pyx_k_MB_START, sizeof(__pyx_k_MB_START), 0, 0, 1, 1}, {&__pyx_kp_u_Non_native_byte_order_not_suppor, __pyx_k_Non_native_byte_order_not_suppor, sizeof(__pyx_k_Non_native_byte_order_not_suppor), 0, 1, 0, 0}, {&__pyx_n_s_PY2, __pyx_k_PY2, sizeof(__pyx_k_PY2), 0, 0, 1, 1}, - {&__pyx_n_s_Pyx_CFunc_object____object___t, __pyx_k_Pyx_CFunc_object____object___t, sizeof(__pyx_k_Pyx_CFunc_object____object___t), 0, 0, 1, 1}, {&__pyx_n_s_RuntimeError, __pyx_k_RuntimeError, sizeof(__pyx_k_RuntimeError), 0, 0, 1, 1}, {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, {&__pyx_kp_u__2, __pyx_k__2, sizeof(__pyx_k__2), 0, 1, 0, 0}, - {&__pyx_n_s_b, __pyx_k_b, sizeof(__pyx_k_b), 0, 0, 1, 1}, - {&__pyx_n_s_cfunc_to_py, __pyx_k_cfunc_to_py, sizeof(__pyx_k_cfunc_to_py), 0, 0, 1, 1}, {&__pyx_n_s_cline_in_traceback, __pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 0, 1, 1}, {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, - {&__pyx_n_s_is_utf8_continue, __pyx_k_is_utf8_continue, sizeof(__pyx_k_is_utf8_continue), 0, 0, 1, 1}, {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, {&__pyx_n_s_max_n, __pyx_k_max_n, sizeof(__pyx_k_max_n), 0, 0, 1, 1}, {&__pyx_n_s_min_n, __pyx_k_min_n, sizeof(__pyx_k_min_n), 0, 0, 1, 1}, @@ -6310,12 +5336,10 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, {&__pyx_kp_s_s, __pyx_k_s, sizeof(__pyx_k_s), 0, 0, 1, 0}, {&__pyx_n_s_six, __pyx_k_six, sizeof(__pyx_k_six), 0, 0, 1, 1}, - {&__pyx_kp_s_stringsource, __pyx_k_stringsource, sizeof(__pyx_k_stringsource), 0, 0, 1, 0}, {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, {&__pyx_kp_u_unknown_dtype_code_in_numpy_pxd, __pyx_k_unknown_dtype_code_in_numpy_pxd, sizeof(__pyx_k_unknown_dtype_code_in_numpy_pxd), 0, 1, 0, 0}, {&__pyx_kp_s_utf_8, __pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 0, 1, 0}, {&__pyx_n_s_word, __pyx_k_word, sizeof(__pyx_k_word), 0, 0, 1, 1}, - {&__pyx_n_s_wrap, __pyx_k_wrap, sizeof(__pyx_k_wrap), 0, 0, 1, 1}, {0, 0, 0, 0, 0, 0, 0} }; static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(void) { @@ -6408,18 +5432,6 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __pyx_tuple__9 = PyTuple_Pack(1, __pyx_kp_s_numpy_core_umath_failed_to_impor); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(1, 1044, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__9); __Pyx_GIVEREF(__pyx_tuple__9); - - /* "cfunc.to_py":65 - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): - * def wrap(object b): # <<<<<<<<<<<<<< - * """wrap(b)""" - * return f(b) - */ - __pyx_tuple__10 = PyTuple_Pack(1, __pyx_n_s_b); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(2, 65, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__10); - __Pyx_GIVEREF(__pyx_tuple__10); - __pyx_codeobj__11 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__10, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_stringsource, __pyx_n_s_wrap, 65, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__11)) __PYX_ERR(2, 65, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -6431,8 +5443,6 @@ static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void) { if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(0, 1, __pyx_L1_error); __pyx_int_0 = PyInt_FromLong(0); if (unlikely(!__pyx_int_0)) __PYX_ERR(0, 1, __pyx_L1_error) __pyx_int_1 = PyInt_FromLong(1); if (unlikely(!__pyx_int_1)) __PYX_ERR(0, 1, __pyx_L1_error) - __pyx_int_128 = PyInt_FromLong(128); if (unlikely(!__pyx_int_128)) __PYX_ERR(0, 1, __pyx_L1_error) - __pyx_int_192 = PyInt_FromLong(192); if (unlikely(!__pyx_int_192)) __PYX_ERR(0, 1, __pyx_L1_error) return 0; __pyx_L1_error:; return -1; @@ -6474,17 +5484,8 @@ static int __Pyx_modinit_type_init_code(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); /*--- Type init code ---*/ - if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(2, 64, __pyx_L1_error) - __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_print = 0; - if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_dictoffset && __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_getattro == PyObject_GenericGetAttr)) { - __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; - } - __pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py = &__pyx_scope_struct____Pyx_CFunc_object____object___to_py; __Pyx_RefNannyFinishContext(); return 0; - __pyx_L1_error:; - __Pyx_RefNannyFinishContext(); - return -1; } static int __Pyx_modinit_type_import_code(void) { @@ -6492,7 +5493,7 @@ static int __Pyx_modinit_type_import_code(void) { PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); /*--- Type import code ---*/ - __pyx_t_1 = PyImport_ImportModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 9, __pyx_L1_error) + __pyx_t_1 = PyImport_ImportModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 9, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_ptype_7cpython_4type_type = __Pyx_ImportType(__pyx_t_1, __Pyx_BUILTIN_MODULE_NAME, "type", #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 @@ -6501,7 +5502,7 @@ static int __Pyx_modinit_type_import_code(void) { sizeof(PyHeapTypeObject), #endif __Pyx_ImportType_CheckSize_Warn); - if (!__pyx_ptype_7cpython_4type_type) __PYX_ERR(3, 9, __pyx_L1_error) + if (!__pyx_ptype_7cpython_4type_type) __PYX_ERR(2, 9, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = PyImport_ImportModule("numpy"); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 206, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); @@ -6634,7 +5635,6 @@ static CYTHON_SMALL_CODE int __pyx_pymod_exec__utils_any2vec(PyObject *__pyx_pyi { PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; - int __pyx_t_3; __Pyx_RefNannyDeclarations #if CYTHON_PEP489_MULTI_PHASE_INIT if (__pyx_m) { @@ -6731,7 +5731,7 @@ if (!__Pyx_RefNanny) { (void)__Pyx_modinit_global_init_code(); (void)__Pyx_modinit_variable_export_code(); (void)__Pyx_modinit_function_export_code(); - if (unlikely(__Pyx_modinit_type_init_code() != 0)) goto __pyx_L1_error; + (void)__Pyx_modinit_type_init_code(); if (unlikely(__Pyx_modinit_type_import_code() != 0)) goto __pyx_L1_error; (void)__Pyx_modinit_variable_import_code(); (void)__Pyx_modinit_function_import_code(); @@ -6776,45 +5776,20 @@ if (!__Pyx_RefNanny) { /* "gensim/models/_utils_any2vec.pyx":100 * # as opposed to a new character. * # - * _MB_MASK = 0xC0 # <<<<<<<<<<<<<< - * _MB_START = 0x80 + * cdef unsigned char _MB_MASK = 0xC0 # <<<<<<<<<<<<<< + * cdef unsigned char _MB_START = 0x80 * */ - if (PyDict_SetItem(__pyx_d, __pyx_n_s_MB_MASK, __pyx_int_192) < 0) __PYX_ERR(0, 100, __pyx_L1_error) + __pyx_v_6gensim_6models_14_utils_any2vec__MB_MASK = 0xC0; /* "gensim/models/_utils_any2vec.pyx":101 * # - * _MB_MASK = 0xC0 - * _MB_START = 0x80 # <<<<<<<<<<<<<< - * - * - */ - if (PyDict_SetItem(__pyx_d, __pyx_n_s_MB_START, __pyx_int_128) < 0) __PYX_ERR(0, 101, __pyx_L1_error) - - /* "gensim/models/_utils_any2vec.pyx":112 - * - * - * _is_utf8_continue = _is_utf8_continue_py2 if PY2 else _is_utf8_continue_py3 # <<<<<<<<<<<<<< + * cdef unsigned char _MB_MASK = 0xC0 + * cdef unsigned char _MB_START = 0x80 # <<<<<<<<<<<<<< * * */ - __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_PY2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 112, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 112, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (__pyx_t_3) { - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__is_utf8_continue_py2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 112, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_t_1; - __pyx_t_1 = 0; - } else { - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_6gensim_6models_14_utils_any2vec__is_utf8_continue_py3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 112, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_t_1; - __pyx_t_1 = 0; - } - if (PyDict_SetItem(__pyx_d, __pyx_n_s_is_utf8_continue, __pyx_t_2) < 0) __PYX_ERR(0, 112, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_6gensim_6models_14_utils_any2vec__MB_START = 0x80; /* "gensim/models/_utils_any2vec.pyx":1 * #!/usr/bin/env cython # <<<<<<<<<<<<<< @@ -6826,12 +5801,12 @@ if (!__Pyx_RefNanny) { if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_2) < 0) __PYX_ERR(0, 1, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "cfunc.to_py":64 + /* "../../envs/gensim/lib/python3.7/site-packages/Cython/Includes/numpy/__init__.pxd":1046 + * raise ImportError("numpy.core.umath failed to import") * - * @cname("__Pyx_CFunc_object____object___to_py") - * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< - * def wrap(object b): - * """wrap(b)""" + * cdef inline int import_ufunc() except -1: # <<<<<<<<<<<<<< + * try: + * _import_umath() */ /*--- Wrapped vars code ---*/ @@ -7374,41 +6349,6 @@ static int __Pyx_ParseOptionalKeywords( return -1; } -/* GetModuleGlobalName */ -#if CYTHON_USE_DICT_VERSIONS -static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) -#else -static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) -#endif -{ - PyObject *result; -#if !CYTHON_AVOID_BORROWED_REFS -#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 - result = _PyDict_GetItem_KnownHash(__pyx_d, name, ((PyASCIIObject *) name)->hash); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } else if (unlikely(PyErr_Occurred())) { - return NULL; - } -#else - result = PyDict_GetItem(__pyx_d, name); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } -#endif -#else - result = PyObject_GetItem(__pyx_d, name); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } - PyErr_Clear(); -#endif - return __Pyx_GetBuiltinName(name); -} - /* PyCFunctionFastCall */ #if CYTHON_FAST_PYCCALL static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { @@ -7640,93 +6580,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObjec } #endif -/* GetItemInt */ -static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { - PyObject *r; - if (!j) return NULL; - r = PyObject_GetItem(o, j); - Py_DECREF(j); - return r; -} -static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, - CYTHON_NCP_UNUSED int wraparound, - CYTHON_NCP_UNUSED int boundscheck) { -#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - Py_ssize_t wrapped_i = i; - if (wraparound & unlikely(i < 0)) { - wrapped_i += PyList_GET_SIZE(o); - } - if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyList_GET_SIZE(o)))) { - PyObject *r = PyList_GET_ITEM(o, wrapped_i); - Py_INCREF(r); - return r; - } - return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); -#else - return PySequence_GetItem(o, i); -#endif -} -static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, - CYTHON_NCP_UNUSED int wraparound, - CYTHON_NCP_UNUSED int boundscheck) { -#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - Py_ssize_t wrapped_i = i; - if (wraparound & unlikely(i < 0)) { - wrapped_i += PyTuple_GET_SIZE(o); - } - if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyTuple_GET_SIZE(o)))) { - PyObject *r = PyTuple_GET_ITEM(o, wrapped_i); - Py_INCREF(r); - return r; - } - return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); -#else - return PySequence_GetItem(o, i); -#endif -} -static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, - CYTHON_NCP_UNUSED int wraparound, - CYTHON_NCP_UNUSED int boundscheck) { -#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS && CYTHON_USE_TYPE_SLOTS - if (is_list || PyList_CheckExact(o)) { - Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); - if ((!boundscheck) || (likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o))))) { - PyObject *r = PyList_GET_ITEM(o, n); - Py_INCREF(r); - return r; - } - } - else if (PyTuple_CheckExact(o)) { - Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); - if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyTuple_GET_SIZE(o)))) { - PyObject *r = PyTuple_GET_ITEM(o, n); - Py_INCREF(r); - return r; - } - } else { - PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (likely(m && m->sq_item)) { - if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { - Py_ssize_t l = m->sq_length(o); - if (likely(l >= 0)) { - i += l; - } else { - if (!PyErr_ExceptionMatches(PyExc_OverflowError)) - return NULL; - PyErr_Clear(); - } - } - return m->sq_item(o, i); - } - } -#else - if (is_list || PySequence_Check(o)) { - return PySequence_GetItem(o, i); - } -#endif - return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); -} - /* PyErrFetchRestore */ #if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { @@ -7951,849 +6804,174 @@ static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { static CYTHON_INLINE void __Pyx_RaiseNoneNotIterableError(void) { PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); } - -/* ExtTypeTest */ -static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) { - if (unlikely(!type)) { - PyErr_SetString(PyExc_SystemError, "Missing type object"); - return 0; - } - if (likely(__Pyx_TypeCheck(obj, type))) - return 1; - PyErr_Format(PyExc_TypeError, "Cannot convert %.200s to %.200s", - Py_TYPE(obj)->tp_name, type->tp_name); - return 0; -} - -/* GetTopmostException */ -#if CYTHON_USE_EXC_INFO_STACK -static _PyErr_StackItem * -__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) -{ - _PyErr_StackItem *exc_info = tstate->exc_info; - while ((exc_info->exc_type == NULL || exc_info->exc_type == Py_None) && - exc_info->previous_item != NULL) - { - exc_info = exc_info->previous_item; - } - return exc_info; -} -#endif - -/* SaveResetException */ -#if CYTHON_FAST_THREAD_STATE -static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { - #if CYTHON_USE_EXC_INFO_STACK - _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); - *type = exc_info->exc_type; - *value = exc_info->exc_value; - *tb = exc_info->exc_traceback; - #else - *type = tstate->exc_type; - *value = tstate->exc_value; - *tb = tstate->exc_traceback; - #endif - Py_XINCREF(*type); - Py_XINCREF(*value); - Py_XINCREF(*tb); -} -static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - #if CYTHON_USE_EXC_INFO_STACK - _PyErr_StackItem *exc_info = tstate->exc_info; - tmp_type = exc_info->exc_type; - tmp_value = exc_info->exc_value; - tmp_tb = exc_info->exc_traceback; - exc_info->exc_type = type; - exc_info->exc_value = value; - exc_info->exc_traceback = tb; - #else - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = type; - tstate->exc_value = value; - tstate->exc_traceback = tb; - #endif - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -} -#endif - -/* PyErrExceptionMatches */ -#if CYTHON_FAST_THREAD_STATE -static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { - Py_ssize_t i, n; - n = PyTuple_GET_SIZE(tuple); -#if PY_MAJOR_VERSION >= 3 - for (i=0; icurexc_type; - if (exc_type == err) return 1; - if (unlikely(!exc_type)) return 0; - if (unlikely(PyTuple_Check(err))) - return __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); - return __Pyx_PyErr_GivenExceptionMatches(exc_type, err); -} -#endif - -/* GetException */ -#if CYTHON_FAST_THREAD_STATE -static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) -#else -static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) -#endif -{ - PyObject *local_type, *local_value, *local_tb; -#if CYTHON_FAST_THREAD_STATE - PyObject *tmp_type, *tmp_value, *tmp_tb; - local_type = tstate->curexc_type; - local_value = tstate->curexc_value; - local_tb = tstate->curexc_traceback; - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; -#else - PyErr_Fetch(&local_type, &local_value, &local_tb); -#endif - PyErr_NormalizeException(&local_type, &local_value, &local_tb); -#if CYTHON_FAST_THREAD_STATE - if (unlikely(tstate->curexc_type)) -#else - if (unlikely(PyErr_Occurred())) -#endif - goto bad; - #if PY_MAJOR_VERSION >= 3 - if (local_tb) { - if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) - goto bad; - } - #endif - Py_XINCREF(local_tb); - Py_XINCREF(local_type); - Py_XINCREF(local_value); - *type = local_type; - *value = local_value; - *tb = local_tb; -#if CYTHON_FAST_THREAD_STATE - #if CYTHON_USE_EXC_INFO_STACK - { - _PyErr_StackItem *exc_info = tstate->exc_info; - tmp_type = exc_info->exc_type; - tmp_value = exc_info->exc_value; - tmp_tb = exc_info->exc_traceback; - exc_info->exc_type = local_type; - exc_info->exc_value = local_value; - exc_info->exc_traceback = local_tb; - } - #else - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = local_type; - tstate->exc_value = local_value; - tstate->exc_traceback = local_tb; - #endif - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -#else - PyErr_SetExcInfo(local_type, local_value, local_tb); -#endif - return 0; -bad: - *type = 0; - *value = 0; - *tb = 0; - Py_XDECREF(local_type); - Py_XDECREF(local_value); - Py_XDECREF(local_tb); - return -1; -} - -/* FetchCommonType */ -static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { - PyObject* fake_module; - PyTypeObject* cached_type = NULL; - fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); - if (!fake_module) return NULL; - Py_INCREF(fake_module); - cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); - if (cached_type) { - if (!PyType_Check((PyObject*)cached_type)) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s is not a type object", - type->tp_name); - goto bad; - } - if (cached_type->tp_basicsize != type->tp_basicsize) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s has the wrong size, try recompiling", - type->tp_name); - goto bad; - } - } else { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; - PyErr_Clear(); - if (PyType_Ready(type) < 0) goto bad; - if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) - goto bad; - Py_INCREF(type); - cached_type = type; - } -done: - Py_DECREF(fake_module); - return cached_type; -bad: - Py_XDECREF(cached_type); - cached_type = NULL; - goto done; -} - -/* CythonFunction */ -#include -static PyObject * -__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure) -{ - if (unlikely(op->func_doc == NULL)) { - if (op->func.m_ml->ml_doc) { -#if PY_MAJOR_VERSION >= 3 - op->func_doc = PyUnicode_FromString(op->func.m_ml->ml_doc); -#else - op->func_doc = PyString_FromString(op->func.m_ml->ml_doc); -#endif - if (unlikely(op->func_doc == NULL)) - return NULL; - } else { - Py_INCREF(Py_None); - return Py_None; - } - } - Py_INCREF(op->func_doc); - return op->func_doc; -} -static int -__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) -{ - PyObject *tmp = op->func_doc; - if (value == NULL) { - value = Py_None; - } - Py_INCREF(value); - op->func_doc = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - if (unlikely(op->func_name == NULL)) { -#if PY_MAJOR_VERSION >= 3 - op->func_name = PyUnicode_InternFromString(op->func.m_ml->ml_name); -#else - op->func_name = PyString_InternFromString(op->func.m_ml->ml_name); -#endif - if (unlikely(op->func_name == NULL)) - return NULL; - } - Py_INCREF(op->func_name); - return op->func_name; -} -static int -__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) -{ - PyObject *tmp; -#if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) -#else - if (unlikely(value == NULL || !PyString_Check(value))) -#endif - { - PyErr_SetString(PyExc_TypeError, - "__name__ must be set to a string object"); - return -1; - } - tmp = op->func_name; - Py_INCREF(value); - op->func_name = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - Py_INCREF(op->func_qualname); - return op->func_qualname; -} -static int -__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) -{ - PyObject *tmp; -#if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) -#else - if (unlikely(value == NULL || !PyString_Check(value))) -#endif - { - PyErr_SetString(PyExc_TypeError, - "__qualname__ must be set to a string object"); - return -1; - } - tmp = op->func_qualname; - Py_INCREF(value); - op->func_qualname = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_self(__pyx_CyFunctionObject *m, CYTHON_UNUSED void *closure) -{ - PyObject *self; - self = m->func_closure; - if (self == NULL) - self = Py_None; - Py_INCREF(self); - return self; -} -static PyObject * -__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - if (unlikely(op->func_dict == NULL)) { - op->func_dict = PyDict_New(); - if (unlikely(op->func_dict == NULL)) - return NULL; - } - Py_INCREF(op->func_dict); - return op->func_dict; -} -static int -__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) -{ - PyObject *tmp; - if (unlikely(value == NULL)) { - PyErr_SetString(PyExc_TypeError, - "function's dictionary may not be deleted"); - return -1; - } - if (unlikely(!PyDict_Check(value))) { - PyErr_SetString(PyExc_TypeError, - "setting function's dictionary to a non-dict"); - return -1; - } - tmp = op->func_dict; - Py_INCREF(value); - op->func_dict = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - Py_INCREF(op->func_globals); - return op->func_globals; -} -static PyObject * -__Pyx_CyFunction_get_closure(CYTHON_UNUSED __pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - Py_INCREF(Py_None); - return Py_None; -} -static PyObject * -__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) -{ - PyObject* result = (op->func_code) ? op->func_code : Py_None; - Py_INCREF(result); - return result; -} -static int -__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { - int result = 0; - PyObject *res = op->defaults_getter((PyObject *) op); - if (unlikely(!res)) - return -1; - #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - op->defaults_tuple = PyTuple_GET_ITEM(res, 0); - Py_INCREF(op->defaults_tuple); - op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); - Py_INCREF(op->defaults_kwdict); - #else - op->defaults_tuple = PySequence_ITEM(res, 0); - if (unlikely(!op->defaults_tuple)) result = -1; - else { - op->defaults_kwdict = PySequence_ITEM(res, 1); - if (unlikely(!op->defaults_kwdict)) result = -1; - } - #endif - Py_DECREF(res); - return result; -} -static int -__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; - if (!value) { - value = Py_None; - } else if (value != Py_None && !PyTuple_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "__defaults__ must be set to a tuple object"); - return -1; - } - Py_INCREF(value); - tmp = op->defaults_tuple; - op->defaults_tuple = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { - PyObject* result = op->defaults_tuple; - if (unlikely(!result)) { - if (op->defaults_getter) { - if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; - result = op->defaults_tuple; - } else { - result = Py_None; - } - } - Py_INCREF(result); - return result; -} -static int -__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; - if (!value) { - value = Py_None; - } else if (value != Py_None && !PyDict_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "__kwdefaults__ must be set to a dict object"); - return -1; - } - Py_INCREF(value); - tmp = op->defaults_kwdict; - op->defaults_kwdict = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { - PyObject* result = op->defaults_kwdict; - if (unlikely(!result)) { - if (op->defaults_getter) { - if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; - result = op->defaults_kwdict; - } else { - result = Py_None; - } - } - Py_INCREF(result); - return result; -} -static int -__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; - if (!value || value == Py_None) { - value = NULL; - } else if (!PyDict_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "__annotations__ must be set to a dict object"); - return -1; - } - Py_XINCREF(value); - tmp = op->func_annotations; - op->func_annotations = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { - PyObject* result = op->func_annotations; - if (unlikely(!result)) { - result = PyDict_New(); - if (unlikely(!result)) return NULL; - op->func_annotations = result; - } - Py_INCREF(result); - return result; -} -static PyGetSetDef __pyx_CyFunction_getsets[] = { - {(char *) "func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, - {(char *) "__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, - {(char *) "func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, - {(char *) "__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, - {(char *) "__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, - {(char *) "__self__", (getter)__Pyx_CyFunction_get_self, 0, 0, 0}, - {(char *) "func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, - {(char *) "__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, - {(char *) "func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, - {(char *) "__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, - {(char *) "func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, - {(char *) "__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, - {(char *) "func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, - {(char *) "__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, - {(char *) "func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, - {(char *) "__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, - {(char *) "__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, - {(char *) "__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, - {0, 0, 0, 0, 0} -}; -static PyMemberDef __pyx_CyFunction_members[] = { - {(char *) "__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), PY_WRITE_RESTRICTED, 0}, - {0, 0, 0, 0, 0} -}; -static PyObject * -__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, CYTHON_UNUSED PyObject *args) -{ -#if PY_MAJOR_VERSION >= 3 - return PyUnicode_FromString(m->func.m_ml->ml_name); -#else - return PyString_FromString(m->func.m_ml->ml_name); -#endif -} -static PyMethodDef __pyx_CyFunction_methods[] = { - {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, - {0, 0, 0, 0} -}; -#if PY_VERSION_HEX < 0x030500A0 -#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) -#else -#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func.m_weakreflist) -#endif -static PyObject *__Pyx_CyFunction_New(PyTypeObject *type, PyMethodDef *ml, int flags, PyObject* qualname, - PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { - __pyx_CyFunctionObject *op = PyObject_GC_New(__pyx_CyFunctionObject, type); - if (op == NULL) - return NULL; - op->flags = flags; - __Pyx_CyFunction_weakreflist(op) = NULL; - op->func.m_ml = ml; - op->func.m_self = (PyObject *) op; - Py_XINCREF(closure); - op->func_closure = closure; - Py_XINCREF(module); - op->func.m_module = module; - op->func_dict = NULL; - op->func_name = NULL; - Py_INCREF(qualname); - op->func_qualname = qualname; - op->func_doc = NULL; - op->func_classobj = NULL; - op->func_globals = globals; - Py_INCREF(op->func_globals); - Py_XINCREF(code); - op->func_code = code; - op->defaults_pyobjects = 0; - op->defaults = NULL; - op->defaults_tuple = NULL; - op->defaults_kwdict = NULL; - op->defaults_getter = NULL; - op->func_annotations = NULL; - PyObject_GC_Track(op); - return (PyObject *) op; -} -static int -__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) -{ - Py_CLEAR(m->func_closure); - Py_CLEAR(m->func.m_module); - Py_CLEAR(m->func_dict); - Py_CLEAR(m->func_name); - Py_CLEAR(m->func_qualname); - Py_CLEAR(m->func_doc); - Py_CLEAR(m->func_globals); - Py_CLEAR(m->func_code); - Py_CLEAR(m->func_classobj); - Py_CLEAR(m->defaults_tuple); - Py_CLEAR(m->defaults_kwdict); - Py_CLEAR(m->func_annotations); - if (m->defaults) { - PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); - int i; - for (i = 0; i < m->defaults_pyobjects; i++) - Py_XDECREF(pydefaults[i]); - PyObject_Free(m->defaults); - m->defaults = NULL; + +/* ExtTypeTest */ +static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) { + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; } + if (likely(__Pyx_TypeCheck(obj, type))) + return 1; + PyErr_Format(PyExc_TypeError, "Cannot convert %.200s to %.200s", + Py_TYPE(obj)->tp_name, type->tp_name); return 0; } -static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) + +/* GetTopmostException */ +#if CYTHON_USE_EXC_INFO_STACK +static _PyErr_StackItem * +__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) { - if (__Pyx_CyFunction_weakreflist(m) != NULL) - PyObject_ClearWeakRefs((PyObject *) m); - __Pyx_CyFunction_clear(m); - PyObject_GC_Del(m); + _PyErr_StackItem *exc_info = tstate->exc_info; + while ((exc_info->exc_type == NULL || exc_info->exc_type == Py_None) && + exc_info->previous_item != NULL) + { + exc_info = exc_info->previous_item; + } + return exc_info; } -static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) -{ - PyObject_GC_UnTrack(m); - __Pyx__CyFunction_dealloc(m); +#endif + +/* SaveResetException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + *type = exc_info->exc_type; + *value = exc_info->exc_value; + *tb = exc_info->exc_traceback; + #else + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + #endif + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); } -static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) -{ - Py_VISIT(m->func_closure); - Py_VISIT(m->func.m_module); - Py_VISIT(m->func_dict); - Py_VISIT(m->func_name); - Py_VISIT(m->func_qualname); - Py_VISIT(m->func_doc); - Py_VISIT(m->func_globals); - Py_VISIT(m->func_code); - Py_VISIT(m->func_classobj); - Py_VISIT(m->defaults_tuple); - Py_VISIT(m->defaults_kwdict); - if (m->defaults) { - PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); - int i; - for (i = 0; i < m->defaults_pyobjects; i++) - Py_VISIT(pydefaults[i]); - } - return 0; +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = type; + exc_info->exc_value = value; + exc_info->exc_traceback = tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); } -static PyObject *__Pyx_CyFunction_descr_get(PyObject *func, PyObject *obj, PyObject *type) -{ - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - if (m->flags & __Pyx_CYFUNCTION_STATICMETHOD) { - Py_INCREF(func); - return func; - } - if (m->flags & __Pyx_CYFUNCTION_CLASSMETHOD) { - if (type == NULL) - type = (PyObject *)(Py_TYPE(obj)); - return __Pyx_PyMethod_New(func, type, (PyObject *)(Py_TYPE(type))); - } - if (obj == Py_None) - obj = NULL; - return __Pyx_PyMethod_New(func, obj, type); -} -static PyObject* -__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) -{ +#endif + +/* PyErrExceptionMatches */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); #if PY_MAJOR_VERSION >= 3 - return PyUnicode_FromFormat("", - op->func_qualname, (void *)op); -#else - return PyString_FromFormat("", - PyString_AsString(op->func_qualname), (void *)op); + for (i=0; im_ml->ml_meth; - Py_ssize_t size; - switch (f->m_ml->ml_flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { - case METH_VARARGS: - if (likely(kw == NULL || PyDict_Size(kw) == 0)) - return (*meth)(self, arg); - break; - case METH_VARARGS | METH_KEYWORDS: - return (*(PyCFunctionWithKeywords)(void*)meth)(self, arg, kw); - case METH_NOARGS: - if (likely(kw == NULL || PyDict_Size(kw) == 0)) { - size = PyTuple_GET_SIZE(arg); - if (likely(size == 0)) - return (*meth)(self, NULL); - PyErr_Format(PyExc_TypeError, - "%.200s() takes no arguments (%" CYTHON_FORMAT_SSIZE_T "d given)", - f->m_ml->ml_name, size); - return NULL; - } - break; - case METH_O: - if (likely(kw == NULL || PyDict_Size(kw) == 0)) { - size = PyTuple_GET_SIZE(arg); - if (likely(size == 1)) { - PyObject *result, *arg0; - #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - arg0 = PyTuple_GET_ITEM(arg, 0); - #else - arg0 = PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; - #endif - result = (*meth)(self, arg0); - #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) - Py_DECREF(arg0); - #endif - return result; - } - PyErr_Format(PyExc_TypeError, - "%.200s() takes exactly one argument (%" CYTHON_FORMAT_SSIZE_T "d given)", - f->m_ml->ml_name, size); - return NULL; - } - break; - default: - PyErr_SetString(PyExc_SystemError, "Bad call flags in " - "__Pyx_CyFunction_Call. METH_OLDARGS is no " - "longer supported!"); - return NULL; + for (i=0; im_ml->ml_name); - return NULL; -} -static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { - return __Pyx_CyFunction_CallMethod(func, ((PyCFunctionObject*)func)->m_self, arg, kw); + return 0; } -static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { - PyObject *result; - __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; - if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { - Py_ssize_t argc; - PyObject *new_args; - PyObject *self; - argc = PyTuple_GET_SIZE(args); - new_args = PyTuple_GetSlice(args, 1, argc); - if (unlikely(!new_args)) - return NULL; - self = PyTuple_GetItem(args, 0); - if (unlikely(!self)) { - Py_DECREF(new_args); - return NULL; - } - result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); - Py_DECREF(new_args); - } else { - result = __Pyx_CyFunction_Call(func, args, kw); - } - return result; +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err) { + PyObject *exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; + if (unlikely(PyTuple_Check(err))) + return __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + return __Pyx_PyErr_GivenExceptionMatches(exc_type, err); } -static PyTypeObject __pyx_CyFunctionType_type = { - PyVarObject_HEAD_INIT(0, 0) - "cython_function_or_method", - sizeof(__pyx_CyFunctionObject), - 0, - (destructor) __Pyx_CyFunction_dealloc, - 0, - 0, - 0, -#if PY_MAJOR_VERSION < 3 - 0, +#endif + +/* GetException */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) #else - 0, -#endif - (reprfunc) __Pyx_CyFunction_repr, - 0, - 0, - 0, - 0, - __Pyx_CyFunction_CallAsMethod, - 0, - 0, - 0, - 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - 0, - (traverseproc) __Pyx_CyFunction_traverse, - (inquiry) __Pyx_CyFunction_clear, - 0, -#if PY_VERSION_HEX < 0x030500A0 - offsetof(__pyx_CyFunctionObject, func_weakreflist), +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) +#endif +{ + PyObject *local_type, *local_value, *local_tb; +#if CYTHON_FAST_THREAD_STATE + PyObject *tmp_type, *tmp_value, *tmp_tb; + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; #else - offsetof(PyCFunctionObject, m_weakreflist), -#endif - 0, - 0, - __pyx_CyFunction_methods, - __pyx_CyFunction_members, - __pyx_CyFunction_getsets, - 0, - 0, - __Pyx_CyFunction_descr_get, - 0, - offsetof(__pyx_CyFunctionObject, func_dict), - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, -#if PY_VERSION_HEX >= 0x030400a1 - 0, + PyErr_Fetch(&local_type, &local_value, &local_tb); #endif -}; -static int __pyx_CyFunction_init(void) { - __pyx_CyFunctionType = __Pyx_FetchCommonType(&__pyx_CyFunctionType_type); - if (unlikely(__pyx_CyFunctionType == NULL)) { - return -1; - } - return 0; -} -static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *func, size_t size, int pyobjects) { - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - m->defaults = PyObject_Malloc(size); - if (unlikely(!m->defaults)) - return PyErr_NoMemory(); - memset(m->defaults, 0, size); - m->defaults_pyobjects = pyobjects; - return m->defaults; -} -static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - m->defaults_tuple = tuple; - Py_INCREF(tuple); -} -static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - m->defaults_kwdict = dict; - Py_INCREF(dict); -} -static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - m->func_annotations = dict; - Py_INCREF(dict); -} - -/* PyObject_GenericGetAttrNoDict */ -#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 -static PyObject *__Pyx_RaiseGenericGetAttributeError(PyTypeObject *tp, PyObject *attr_name) { - PyErr_Format(PyExc_AttributeError, -#if PY_MAJOR_VERSION >= 3 - "'%.50s' object has no attribute '%U'", - tp->tp_name, attr_name); + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_FAST_THREAD_STATE + if (unlikely(tstate->curexc_type)) #else - "'%.50s' object has no attribute '%.400s'", - tp->tp_name, PyString_AS_STRING(attr_name)); + if (unlikely(PyErr_Occurred())) #endif - return NULL; -} -static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name) { - PyObject *descr; - PyTypeObject *tp = Py_TYPE(obj); - if (unlikely(!PyString_Check(attr_name))) { - return PyObject_GenericGetAttr(obj, attr_name); - } - assert(!tp->tp_dictoffset); - descr = _PyType_Lookup(tp, attr_name); - if (unlikely(!descr)) { - return __Pyx_RaiseGenericGetAttributeError(tp, attr_name); + goto bad; + #if PY_MAJOR_VERSION >= 3 + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; } - Py_INCREF(descr); - #if PY_MAJOR_VERSION < 3 - if (likely(PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_HAVE_CLASS))) #endif + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_FAST_THREAD_STATE + #if CYTHON_USE_EXC_INFO_STACK { - descrgetfunc f = Py_TYPE(descr)->tp_descr_get; - if (unlikely(f)) { - PyObject *res = f(descr, obj, (PyObject *)tp); - Py_DECREF(descr); - return res; - } + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = local_type; + exc_info->exc_value = local_value; + exc_info->exc_traceback = local_tb; } - return descr; -} + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); #endif + return 0; +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +} /* TypeImport */ #ifndef __PYX_HAVE_RT_ImportType @@ -9155,96 +7333,34 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, func_type zero = 0;\ if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ return (target_type) -1;\ - if (is_unsigned && unlikely(value < zero))\ - goto raise_neg_overflow;\ - else\ - goto raise_overflow;\ - }\ - }\ - return (target_type) value;\ - } - -/* CIntToPy */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint32_t(uint32_t value) { - const uint32_t neg_one = (uint32_t) ((uint32_t) 0 - (uint32_t) 1), const_zero = (uint32_t) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(uint32_t) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(uint32_t) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(uint32_t) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(uint32_t) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(uint32_t) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(uint32_t), - little, !is_unsigned); - } -} - -/* CIntToPy */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value) { - const unsigned int neg_one = (unsigned int) ((unsigned int) 0 - (unsigned int) 1), const_zero = (unsigned int) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(unsigned int) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(unsigned int) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(unsigned int) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(unsigned int) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(unsigned int) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(unsigned int), - little, !is_unsigned); + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ } -} /* CIntToPy */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { - const long neg_one = (long) ((long) 0 - (long) 1), const_zero = (long) 0; +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint32_t(uint32_t value) { + const uint32_t neg_one = (uint32_t) ((uint32_t) 0 - (uint32_t) 1), const_zero = (uint32_t) 0; const int is_unsigned = neg_one > const_zero; if (is_unsigned) { - if (sizeof(long) < sizeof(long)) { + if (sizeof(uint32_t) < sizeof(long)) { return PyInt_FromLong((long) value); - } else if (sizeof(long) <= sizeof(unsigned long)) { + } else if (sizeof(uint32_t) <= sizeof(unsigned long)) { return PyLong_FromUnsignedLong((unsigned long) value); #ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + } else if (sizeof(uint32_t) <= sizeof(unsigned PY_LONG_LONG)) { return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); #endif } } else { - if (sizeof(long) <= sizeof(long)) { + if (sizeof(uint32_t) <= sizeof(long)) { return PyInt_FromLong((long) value); #ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + } else if (sizeof(uint32_t) <= sizeof(PY_LONG_LONG)) { return PyLong_FromLongLong((PY_LONG_LONG) value); #endif } @@ -9252,30 +7368,30 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { { int one = 1; int little = (int)*(unsigned char *)&one; unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(long), + return _PyLong_FromByteArray(bytes, sizeof(uint32_t), little, !is_unsigned); } } /* CIntToPy */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { - const int neg_one = (int) ((int) 0 - (int) 1), const_zero = (int) 0; +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value) { + const unsigned int neg_one = (unsigned int) ((unsigned int) 0 - (unsigned int) 1), const_zero = (unsigned int) 0; const int is_unsigned = neg_one > const_zero; if (is_unsigned) { - if (sizeof(int) < sizeof(long)) { + if (sizeof(unsigned int) < sizeof(long)) { return PyInt_FromLong((long) value); - } else if (sizeof(int) <= sizeof(unsigned long)) { + } else if (sizeof(unsigned int) <= sizeof(unsigned long)) { return PyLong_FromUnsignedLong((unsigned long) value); #ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + } else if (sizeof(unsigned int) <= sizeof(unsigned PY_LONG_LONG)) { return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); #endif } } else { - if (sizeof(int) <= sizeof(long)) { + if (sizeof(unsigned int) <= sizeof(long)) { return PyInt_FromLong((long) value); #ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + } else if (sizeof(unsigned int) <= sizeof(PY_LONG_LONG)) { return PyLong_FromLongLong((PY_LONG_LONG) value); #endif } @@ -9283,7 +7399,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { { int one = 1; int little = (int)*(unsigned char *)&one; unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(int), + return _PyLong_FromByteArray(bytes, sizeof(unsigned int), little, !is_unsigned); } } @@ -9598,6 +7714,37 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { #endif #endif +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { + const int neg_one = (int) ((int) 0 - (int) 1), const_zero = (int) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(int) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(int) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(int), + little, !is_unsigned); + } +} + /* CIntToPy */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_enum__NPY_TYPES(enum NPY_TYPES value) { const enum NPY_TYPES neg_one = (enum NPY_TYPES) ((enum NPY_TYPES) 0 - (enum NPY_TYPES) 1), const_zero = (enum NPY_TYPES) 0; @@ -9926,39 +8073,228 @@ static CYTHON_INLINE char __Pyx_PyInt_As_char(PyObject *x) { } break; case 3: - if (8 * sizeof(char) > 2 * PyLong_SHIFT) { + if (8 * sizeof(char) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) - 1 > 3 * PyLong_SHIFT) { + return (char) ((((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(char) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) - 1 > 4 * PyLong_SHIFT) { + return (char) (((char)-1)*(((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(char) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(char) - 1 > 4 * PyLong_SHIFT) { + return (char) ((((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + } +#endif + if (sizeof(char) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(char, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(char) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(char, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + char val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (char) -1; + } + } else { + char val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (char) -1; + val = __Pyx_PyInt_As_char(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to char"); + return (char) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to char"); + return (char) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { + const size_t neg_one = (size_t) ((size_t) 0 - (size_t) 1), const_zero = (size_t) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(size_t) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(size_t, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (size_t) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (size_t) 0; + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, digits[0]) + case 2: + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 2 * PyLong_SHIFT) { + return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 3 * PyLong_SHIFT) { + return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 4 * PyLong_SHIFT) { + return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (size_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(size_t) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (size_t) 0; + case -1: __PYX_VERIFY_RETURN_INT(size_t, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, +digits[0]) + case -2: + if (8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(char) - 1 > 3 * PyLong_SHIFT) { - return (char) ((((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); } } break; case -4: - if (8 * sizeof(char) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(char) - 1 > 4 * PyLong_SHIFT) { - return (char) (((char)-1)*(((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); } } break; case 4: - if (8 * sizeof(char) > 3 * PyLong_SHIFT) { + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(char) - 1 > 4 * PyLong_SHIFT) { - return (char) ((((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); } } break; } #endif - if (sizeof(char) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(char, long, PyLong_AsLong(x)) + if (sizeof(size_t) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof(char) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(char, PY_LONG_LONG, PyLong_AsLongLong(x)) + } else if (sizeof(size_t) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) #endif } } @@ -9967,7 +8303,7 @@ static CYTHON_INLINE char __Pyx_PyInt_As_char(PyObject *x) { PyErr_SetString(PyExc_RuntimeError, "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); #else - char val; + size_t val; PyObject *v = __Pyx_PyNumber_IntOrLong(x); #if PY_MAJOR_VERSION < 3 if (likely(v) && !PyLong_Check(v)) { @@ -9987,24 +8323,24 @@ static CYTHON_INLINE char __Pyx_PyInt_As_char(PyObject *x) { return val; } #endif - return (char) -1; + return (size_t) -1; } } else { - char val; + size_t val; PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (char) -1; - val = __Pyx_PyInt_As_char(tmp); + if (!tmp) return (size_t) -1; + val = __Pyx_PyInt_As_size_t(tmp); Py_DECREF(tmp); return val; } raise_overflow: PyErr_SetString(PyExc_OverflowError, - "value too large to convert to char"); - return (char) -1; + "value too large to convert to size_t"); + return (size_t) -1; raise_neg_overflow: PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to char"); - return (char) -1; + "can't convert negative value to size_t"); + return (size_t) -1; } /* CIntFromPy */ @@ -10196,193 +8532,35 @@ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { return (int) -1; } -/* CIntFromPy */ -static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { - const size_t neg_one = (size_t) ((size_t) 0 - (size_t) 1), const_zero = (size_t) 0; +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { + const long neg_one = (long) ((long) 0 - (long) 1), const_zero = (long) 0; const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(size_t) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(size_t, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (size_t) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (size_t) 0; - case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, digits[0]) - case 2: - if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) >= 2 * PyLong_SHIFT) { - return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) >= 3 * PyLong_SHIFT) { - return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) >= 4 * PyLong_SHIFT) { - return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (size_t) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(size_t) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (size_t) 0; - case -1: __PYX_VERIFY_RETURN_INT(size_t, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, +digits[0]) - case -2: - if (8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { - return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { - return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { - return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { - return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { - return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { - return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - } -#endif - if (sizeof(size_t) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); #ifdef HAVE_LONG_LONG - } else if (sizeof(size_t) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); #endif - } } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - size_t val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); #endif - return (size_t) -1; } - } else { - size_t val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (size_t) -1; - val = __Pyx_PyInt_As_size_t(tmp); - Py_DECREF(tmp); - return val; } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to size_t"); - return (size_t) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to size_t"); - return (size_t) -1; + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); + } } /* CIntFromPy */ diff --git a/gensim/models/_utils_any2vec.pyx b/gensim/models/_utils_any2vec.pyx index 9c5965cd4b..cc4ba9bbb4 100644 --- a/gensim/models/_utils_any2vec.pyx +++ b/gensim/models/_utils_any2vec.pyx @@ -97,19 +97,8 @@ cpdef compute_ngrams(word, unsigned int min_n, unsigned int max_n): # UTF-8 bytes that begin with 10 are subsequent bytes of a multi-byte sequence, # as opposed to a new character. # -_MB_MASK = 0xC0 -_MB_START = 0x80 - - -cdef _is_utf8_continue_py3(b): - return b & _MB_MASK == _MB_START - - -cdef _is_utf8_continue_py2(b): - return ord(b) & _MB_MASK == _MB_START - - -_is_utf8_continue = _is_utf8_continue_py2 if PY2 else _is_utf8_continue_py3 +cdef unsigned char _MB_MASK = 0xC0 +cdef unsigned char _MB_START = 0x80 cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): @@ -137,43 +126,19 @@ cpdef compute_ngrams_bytes(word, unsigned int min_n, unsigned int max_n): """ cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") - cdef int num_bytes = len(utf8_word) - cdef int j, i, n - - ngrams = [] - for i in range(num_bytes): - if _is_utf8_continue(utf8_word[i]): - continue - - j, n = i, 1 - while j < num_bytes and n <= max_n: - j += 1 - while j < num_bytes and _is_utf8_continue(utf8_word[j]): - j += 1 - if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): - ngram = bytes(utf8_word[i:j]) - ngrams.append(ngram) - n += 1 - return ngrams - - -cpdef compute_ngrams_bytes_optimized(word, unsigned int min_n, unsigned int max_n): - cdef unsigned char _mb_mask = 0xC0 - cdef unsigned char _mb_start = 0x80 - cdef bytes utf8_word = ('<%s>' % word).encode("utf-8") cdef const unsigned char *bytez = utf8_word cdef size_t num_bytes = len(utf8_word) cdef size_t j, i, n ngrams = [] for i in range(num_bytes): - if bytez[i] & _mb_mask == _mb_start: + if bytez[i] & _MB_MASK == _MB_START: continue j, n = i, 1 while j < num_bytes and n <= max_n: j += 1 - while j < num_bytes and (bytez[j] & _mb_mask) == _mb_start: + while j < num_bytes and (bytez[j] & _MB_MASK) == _MB_START: j += 1 if n >= min_n and not (n == 1 and (i == 0 or j == num_bytes)): ngram = bytes(bytez[i:j]) From 38146bdb2d62a763e6668773c59399ee63009b65 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Thu, 24 Jan 2019 12:44:14 +1100 Subject: [PATCH 31/32] removing temporary benchmarking code --- bench.py | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 bench.py diff --git a/bench.py b/bench.py deleted file mode 100644 index dcec83ddb7..0000000000 --- a/bench.py +++ /dev/null @@ -1,13 +0,0 @@ -import gensim.downloader as api -import itertools as it -from gensim.models.utils_any2vec import compute_ngrams_bytes, ft_hash_bytes - -words = tuple(it.chain.from_iterable(api.load("text8"))) -assert len(words) == 17005207 # long enough - -words = words[:100000] - - -def benchmark(words=words, ngram_func=compute_ngrams_bytes, hash_func=ft_hash_bytes): - for w in words: - arr = [hash_func(ngram) % 10000 for ngram in ngram_func(w, 3, 6)] From 136dc85db3deaf306a8125fc5d835db29a429a10 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Thu, 24 Jan 2019 14:12:05 +1100 Subject: [PATCH 32/32] remove wide characters from fb test code --- gensim/test/test_data/fb-ngrams.txt | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/gensim/test/test_data/fb-ngrams.txt b/gensim/test/test_data/fb-ngrams.txt index bb22c620a3..3296d85c6a 100644 --- a/gensim/test/test_data/fb-ngrams.txt +++ b/gensim/test/test_data/fb-ngrams.txt @@ -87,17 +87,3 @@ he> -0.046122 -0.071463 0.20759 0.25732 0.23174 <試し> 0.11952 0.15093 -0.075759 0.18327 0.12471 試し> -0.14656 0.072651 -0.16777 0.20026 0.21199 - -🚑🚒🚓🚕 -🚑🚒🚓🚕 0 0 0 0 0 -<🚑🚒 -0.065364 0.17951 -0.17059 0.11436 -0.12783 -<🚑🚒🚓 -0.13712 0.028013 -0.041927 0.11244 0.12261 -<🚑🚒🚓🚕 0.18643 0.17268 0.024482 0.044628 -0.14716 -<🚑🚒🚓🚕> -0.17261 0.086891 0.21772 0.10036 0.19114 -🚑🚒🚓 0.11122 -0.16219 -0.15235 0.21407 0.14858 -🚑🚒🚓🚕 -0.15504 0.11166 -0.16434 0.089162 -0.16199 -🚑🚒🚓🚕> -0.13712 0.028013 -0.041927 0.11244 0.12261 -🚒🚓🚕 -0.20296 -0.11805 0.20277 0.063027 -0.041727 -🚒🚓🚕> 0.13526 -0.023216 -0.10788 0.24512 -0.022174 -🚓🚕> -0.1776 0.19906 -0.15572 0.16449 -0.12887 -