Warn when BM25.avgdl < 0

gensim/summarization/bm25.py

@@ -37,6 +37,7 @@
 """
 
 
+import logging
 import math
 from six import iteritems
 from six.moves import range
@@ -48,6 +49,8 @@
 PARAM_B = 0.75
 EPSILON = 0.25
 
+logger = logging.getLogger(__name__)
+
 
 class BM25(object):
     """Implementation of Best Matching 25 ranking function.
@@ -103,6 +106,13 @@ def _initialize(self, corpus):
                 nd[word] += 1
 
         self.avgdl = float(num_doc) / self.corpus_size
+        if self.avgdl < 0:
+            logger.warning(
+                'Average inverse document frequency is less than zero. Your corpus of {} documents'
+                ' is either too small or it does not originate from actual text documents. BM25'
+                ' scores will likely produce a "wrong" ordering of documents.'
+            );
+
         # collect idf sum to calculate an average idf for epsilon value
         idf_sum = 0
         # collect words with negative idf to set them a special epsilon value.