text_analysis.py

from __future__ import division
import os
import pickle
import re
from bs4 import BeautifulSoup
from numpy import std
import spacy


# pre-load and pre-compile required variables and methods
nlp = spacy.load('en_core_web_sm')

html_div_br_div_re = re.compile('</div><div><br></div>')
html_newline_re = re.compile('(<br|</div|</p)')
quotation_re = re.compile(u'[\u00AB\u00BB\u201C\u201D\u201E\u201F\u2033\u2036\u301D\u301E]')
apostrophe_re = re.compile(u'[\u02BC\u2019\u2032]')
nominalization_re = re.compile('(?:ion|ions|ism|isms|ty|ties|ment|ments|ness|nesses|ance|ances|ence|ences)$')

# os.getenv('HOME')
# print(os.getcwd()) = /app
synonyms_path = os.path.join(os.getcwd(), 'nlp_data/synonyms.pickle')
base_lemmas_path = os.path.join(os.getcwd(), 'nlp_data/base_lemmas.pickle') 
phonemes_path = os.path.join(os.getcwd(), 'nlp_data/phonemes.pickle')  
stop_words_path = os.path.join(os.getcwd(), 'nlp_data/stop_words.pickle') 
frequent_words_pickle_path = os.path.join(os.getcwd(), 'nlp_data/frequent_words.pickle') 
filler_words_path = os.path.join(os.getcwd(), 'nlp_data/filler_words.pickle') 

with open(synonyms_path, 'rb') as file:
  dict_synonyms = pickle.load(file)
with open(base_lemmas_path, 'rb') as file:
  dict_base_lemmas = pickle.load(file)
with open(phonemes_path, 'rb') as file:
  dict_phonemes = pickle.load(file)
with open(stop_words_path,'rb') as file:
  dict_stop_words = pickle.load(file)
  # print(dict_stop_words)
# type(nlp.Defaults.stop_words) = <class 'set'> (sure)
nlp.Defaults.stop_words = dict_stop_words
with open(frequent_words_pickle_path,'rb') as file:
  dict_word_freq = pickle.load(file)
with open(filler_words_path, 'rb') as file:
  dict_fillers = pickle.load(file)

dict_weak_verbs = {'is', 'are', 'am','been', 'be','were', 'was'}
dict_entity_substitutions = {'this', 'that', 'you', 'he', 'she', 'it'}
# with open(os.path.join(os.environ.get('NLP_DATA'), 'weak-verbs')) as file:
#   dict_weak_verbs = file.read().splitlines()
#     # weak-verbs is a collection: overused vague verbs
# with open(os.path.join(os.environ.get('NLP_DATA'), 'entity-substitutions')) as file:
#   dict_entity_substitutions = file.read().splitlines()
#   # dict_entity_substitutions is a collection: pronouns and vague determiners


# dict_synonyms and dict_stop_words have '-'

def is_word(token):
  return token.text[0].isalnum() or (token.pos_ in ['VERB', 'PRON'])  # sometimes words are shortened to non-alnum strings like "'s"

def analyze_text(html):
  # create data and metrics dictionaries
  data = dict()
  metrics = dict()

  ### parse text/html string

  # strip html tags
  html = html_div_br_div_re.sub(r'</div>\n', html)
  html = html_newline_re.sub(lambda m: '\n'+m.group(0), html)
  soup = BeautifulSoup(html, 'html.parser')
  original_text = soup.get_text().rstrip('\n')

  # standardize all quotation marks
  # this will make spaCy sentence segmentation more stable
  text = quotation_re.sub('"', original_text)
  text = apostrophe_re.sub("'", text)

  # parse text
  doc = nlp(text)
  # type(doc) = <class 'spacy.tokens.doc.Doc'>
  # type(token.lower_) = str
  
  data['values'] = [token.text for token in doc]
  data['parts_of_speech'] = [token.tag_ for token in doc]
  data['pos_high_level'] = [token.pos_ for token in doc]
  data['syntax_relations'] = [token.dep_ for token in doc]
  data['words'] = [is_word(token) for token in doc]
  data['punctuation_marks'] = [token.is_punct for token in doc]
  data['numbers_of_characters'] = [len(token.text) if is_word(token) else None for token in doc]
  data['stopwords'] = [(token.lower_ in dict_stop_words) if is_word(token) else None for token in doc]

  # find sentences
  # need to manually merge some of sentences identified by spaCy to make sure all of them end with an appropriate punctuation mark
  sents = []
  sents_end_punct = []
  sents_start_idx = []
  new_sent = []
  for sent in doc.sents:
    if not new_sent:
      sents_start_idx.append(sent.start)
    new_sent.extend(sent)
    for token in reversed(sent):
      if token.text[0] in ['.', '?', '!']:  # need to account for some sentences ending in several punctuation marks: "?!", "?-", etc
        sents.append(new_sent)
        sents_end_punct.append(token.text[0])
        new_sent = []
        break
  if new_sent:
    sents.append(new_sent)
    sents_end_punct.append(None)
  data['sentence_numbers'] = [(idx+1) for idx, sent in enumerate(sents) for token in sent]
  data['sentence_end_punctuations'] = [sents_end_punct[idx] for idx, sent in enumerate(sents) for token in sent]

  # find subjects, predicates, clauses and sentence types based on syntactic structure
  data['principal_parts'] = [None] * len(doc)
  data['independent_principal_parts'] = [None] * len(doc)
  sents_types = ['simple'] * len(sents)
  sents_clauses = [0] * len(sents)
  subj_tags = ['nsubj', 'nsubjpass', 'expl']
  for idx, sent in enumerate(sents):
    is_compound = False
    is_complex = False
    roots = [token for token in sent if (token.dep_ == 'ROOT') and (token.pos_ == 'VERB')]
    if len(roots) is 0:
      sents_types[idx] = 'fragment'
      sents_clauses[idx] = 1
      continue
    for root in roots:
      data['independent_principal_parts'][root.i] = 'predicate'
      data['principal_parts'][root.i] = 'predicate'
      sents_clauses[idx] += 1
    for subj in [token for token in sent if (token.dep_ in subj_tags) and (token.head.pos_ == 'VERB')]:
      data['principal_parts'][subj.i] = 'subject'
      if subj.head.dep_ == 'ROOT':
        data['independent_principal_parts'][subj.i] = 'subject'
      else:
        data['principal_parts'][subj.head.i] = 'predicate'
        sents_clauses[idx] += 1
        if subj.head.dep_ in ['parataxis', 'conj']:
          is_compound = True
        else:
          is_complex = True
    for conj in [token for token in sent if (token.dep_ == 'conj') and not data['principal_parts'][token.i]]:
        data['principal_parts'][conj.i] = data['principal_parts'][conj.head.i]
        data['independent_principal_parts'][conj.i] = data['independent_principal_parts'][conj.head.i]
    if is_complex and is_compound:
      sents_types[idx] = 'complex-compound'
    elif is_compound:
      sents_types[idx] = 'compound'
    elif is_complex:
      sents_types[idx] = 'complex'
  data['sentence_types'] = [sents_types[idx] for idx, sent in enumerate(sents) for token in sent]

  # find words
  words = []
  word2token_map = []
  for idx, token in enumerate(doc):
    # idx, token  = (0, '...')(1, '...')(2, '...')... respectively
# type(doc) = spacy.tokens.doc.Doc
# type(data['...'][idx]) = word = child of type(list<token.lower>) (words)
    if data['words'][idx]:
      words.append(token.lower_)
      word2token_map.append(idx)

  # find lemmas
  data['lemmas'] = [token.lemma_ if is_word(token) else None for token in doc]
  for idx, lemma in enumerate(data['lemmas']):
    if lemma == '-PRON-': # spaCy replaces pronouns with '-PRON-' so we need to fix it
      data['lemmas'][idx] = data['values'][idx].lower()

  # find groups of verbs making up predicates & auxiliary verbs within them
  data['verb_groups'] = [None] * len(doc)
  data['auxiliary_verbs'] = [False if is_word(token) else None for token in doc]
  verb_group_stack = []
  verb_group_count = 0
  for idx, token in enumerate(doc):
    if not verb_group_stack:
      if token.text in ["be", "am", "'m", "is", "are", "'re", "was", "were", "will", "'ll", "wo", "have", "'ve", "has", "had", "'d"]:
        verb_group_stack.append(idx)
      elif (token.text is "'s") and (token.pos_ is 'VERB'):
        verb_group_stack.append(idx)
    elif token.text in ['be', 'been', 'being', 'have', 'had']:
      verb_group_stack.append(idx)
    elif data['pos_high_level'][idx] == 'VERB':
      verb_group_stack.append(idx)
      verb_group_count += 1
      for i in verb_group_stack:
        data['verb_groups'][i] = verb_group_count
      for j in verb_group_stack[:-1]:
        data['auxiliary_verbs'][j] = True
      verb_group_stack = []
    elif data['parts_of_speech'][idx][:2] not in ['RB', 'PD']:
      if len(verb_group_stack) > 1:
        verb_group_count += 1
        for i in verb_group_stack:
          data['verb_groups'][i] = verb_group_count
        for j in verb_group_stack[:-1]:
          data['auxiliary_verbs'][j] = True
      verb_group_stack = []

  # find synonyms and base lemmas
  data['synonyms'] = [None] * len(doc)
  data['base_lemmas'] = [None] * len(doc)
  pos_map = {'NN': 'nouns', 'JJ': 'adjectives', 'VB': 'verbs', 'RB': 'adverbs'}
  for idx_word, word in enumerate(words):
    idx = word2token_map[idx_word]
    pos = data['parts_of_speech'][idx][:2]
    # type(dict_synonyms) = dict[str, dict]
    if (pos in pos_map) and (word in dict_synonyms[pos_map[pos]]):
      data['synonyms'][idx] = dict_synonyms[pos_map[pos]][word]
    else:
      data['synonyms'][idx] = []
    if data['lemmas'][idx] in dict_base_lemmas:
      data['base_lemmas'][idx] = dict_base_lemmas[data['lemmas'][idx]]
    else:
      data['base_lemmas'][idx] = data['lemmas'][idx]

  # find expected word frequencies
  data['expected_word_frequencies'] = [0 if is_word(token) else None for token in doc]
  for idx_word, word in enumerate(words):
    idx = word2token_map[idx_word]
    # here
    lemma = data['lemmas'][idx]
    base_lemma = data['base_lemmas'][idx]
    if lemma in dict_word_freq:
      data['expected_word_frequencies'][idx] = dict_word_freq[lemma]
    elif base_lemma in dict_word_freq:
      data['expected_word_frequencies'][idx] = dict_word_freq[base_lemma]

  ### compute metrics on parsed data

  # count number of sentences
  metrics['sentence_count'] = len(sents)

  # count number of words
  metrics['word_count'] = len(words)

  # count number of words per sentence and its standard deviation
  sents_words = [[token.lower_ for token in sent if is_word(token)] for sent in sents]
  sents_length = [len(sent) for sent in sents_words]
  if len(sents_length):
    metrics['words_per_sentence'] = sum(sents_length) / len(sents_length)
  else:
    metrics['words_per_sentence'] = 0
  if len(sents_length) >= 10:
    metrics['std_of_words_per_sentence'] = std(sents_length)
  else:
    metrics['std_of_words_per_sentence'] = -1

  # find extra long and short sentences
  if len(sents_length):
    metrics['long_sentences_ratio'] = len([1 for sent_length in sents_length if sent_length >= 40]) / len(sents_length)
    metrics['short_sentences_ratio'] = len([1 for sent_length in sents_length if sent_length <= 6]) / len(sents_length)
  else:
    metrics['long_sentences_ratio'] = 0
    metrics['short_sentences_ratio'] = 0

  # find vocabulary size
  metrics['vocabulary_size'] = len(set(data['base_lemmas'])) - 1  # subtract 1 for non-words

  # count sentence types based on ending punctuation mark
  if metrics['sentence_count']:
    metrics['declarative_ratio'] = sents_end_punct.count('.') / metrics['sentence_count']
    metrics['interrogative_ratio'] = sents_end_punct.count('?') / metrics['sentence_count']
    metrics['exclamative_ratio'] = sents_end_punct.count('!') / metrics['sentence_count']
  else:
    metrics['declarative_ratio'] = 0
    metrics['interrogative_ratio'] = 0
    metrics['exclamative_ratio'] = 0

  # count sentence types based on clause structure
  if metrics['sentence_count']:
    metrics['simple_ratio'] = sents_types.count('simple') / metrics['sentence_count']
    metrics['fragment_ratio'] = sents_types.count('fragment') / metrics['sentence_count']
    metrics['complex_ratio'] = sents_types.count('complex') / metrics['sentence_count']
    metrics['compound_ratio'] = sents_types.count('compound') / metrics['sentence_count']
    metrics['complex_compound_ratio'] = sents_types.count('complex-compound') / metrics['sentence_count']
  else:
    metrics['simple_ratio'] = 0
    metrics['fragment_ratio'] = 0
    metrics['complex_ratio'] = 0
    metrics['compound_ratio'] = 0
    metrics['complex_compound_ratio'] = 0

  # compute subject-, predicate- and clause-related metrics
  data['clause_heavy_sentences'] = [False] * len(doc)
  sents_predicate_depth = [0] * len(sents)
  data['late_predicates'] = [False] * len(doc)
  data['detached_subjects'] = [False] * len(doc)
  metrics['many_clauses_ratio'] = 0
  metrics['late_predicates_ratio'] = 0
  metrics['detached_subjects_ratio'] = 0
  sents_with_predicate_count = 0
  for idx, sent in enumerate(sents):
    s1 = sents_start_idx[idx]
    s2 = sents_start_idx[idx]+len(sent)
    syntax_mask = data['independent_principal_parts'][s1:s2]
    if 'predicate' in syntax_mask:
      pred_idx = syntax_mask.index('predicate')
      pred_depth = sum([1 for token in sent[:pred_idx] if is_word(token)]) + 1
      sents_with_predicate_count += 1
    else:
      continue
    sents_predicate_depth[idx] = pred_depth
    if sents_clauses[idx] >= 4:
      data['clause_heavy_sentences'][s1:s2] = [(i != None) for i in data['principal_parts'][s1:s2]]
      metrics['many_clauses_ratio'] += 1
    if pred_depth > 15:
      data['late_predicates'][sents_start_idx[idx]+pred_idx] = True
      metrics['late_predicates_ratio'] += 1
    if 'subject' in syntax_mask:
      subj_idx = syntax_mask.index('subject')
      subj_depth = sum([1 for token in sent[:subj_idx] if is_word(token)])
      if (pred_depth - subj_depth > 8):
        data['detached_subjects'][sents_start_idx[idx]+pred_idx] = True
        data['detached_subjects'][sents_start_idx[idx]+subj_idx] = True
        metrics['detached_subjects_ratio'] += 1
  if sents_with_predicate_count:
    metrics['predicate_depth'] = sum(sents_predicate_depth) / sents_with_predicate_count
    metrics['late_predicates_ratio'] = metrics['late_predicates_ratio'] / sents_with_predicate_count
    metrics['detached_subjects_ratio'] = metrics['detached_subjects_ratio'] / sents_with_predicate_count
  else:
    metrics['predicate_depth'] = 0
  if metrics['sentence_count']:
    metrics['clauses_per_sentence'] = sum(sents_clauses) / metrics['sentence_count']
    metrics['many_clauses_ratio'] = metrics['many_clauses_ratio'] / metrics['sentence_count']
  else:
    metrics['clauses_per_sentence'] = 0
    metrics['many_clauses_ratio'] = 0

  # count number of syllables per word
  cmu_words_count = 0
  cmu_syllables_count = 0
  data['numbers_of_syllables'] = [0 if is_word(token) else None for token in doc]
  for idx, word in enumerate(words):
    if word in dict_phonemes:
      cmu_words_count += 1
      syll_num = len([phoneme for phoneme in dict_phonemes[word] if phoneme[-1].isdigit()])
      cmu_syllables_count += syll_num
      data['numbers_of_syllables'][word2token_map[idx]] = syll_num
  if cmu_words_count:
    metrics['syllables_per_word'] = cmu_syllables_count / cmu_words_count
  else:
    metrics['syllables_per_word'] = 0

  # count number of characters in the whole text
  metrics['character_count'] = len(text)

  # count number of characters per word
  char_count = [len(word) for word in words]
  if metrics['word_count']:
    metrics['characters_per_word'] = sum(char_count) / metrics['word_count']
  else:
    metrics['characters_per_word'] = 0

  # count stopwords
  if metrics['word_count']:
    metrics['stopword_ratio'] = data['stopwords'].count(True) / metrics['word_count']
  else:
    metrics['stopword_ratio'] = 0

  # estimate test readability using Flesch-Kincaid Grade Level test
  if (metrics['word_count'] >= 100) and metrics['words_per_sentence'] and metrics['syllables_per_word']:
    metrics['readability'] = 0.39 * metrics['words_per_sentence'] + 11.8 * metrics['syllables_per_word'] - 15.59
  else:
    metrics['readability'] = 0

  # count number of different parts of speech
  noun_count = 0
  pronoun_count = 0
  pronoun_nonpossesive_count = 0
  verb_count = 0
  adjective_count = 0
  adverb_count = 0
  for tag in data['parts_of_speech']:
    if tag[:2] == 'NN':
      noun_count += 1
    elif tag[:2] in ['PR', 'WP', 'EX']:
      pronoun_count += 1
      if tag in ['PRP', 'WP', 'EX']:
        pronoun_nonpossesive_count += 1
    elif tag[:2] in ['VB', 'MD']:
      verb_count += 1
    elif tag[:2] == 'JJ':
      adjective_count += 1
    elif tag[:2] == 'RB':
      adverb_count += 1
  if metrics['word_count']:
    metrics['noun_ratio'] = noun_count / metrics['word_count']
    metrics['pronoun_ratio'] = pronoun_count / metrics['word_count']
    metrics['verb_ratio'] = verb_count / metrics['word_count']
    metrics['adjective_ratio'] = adjective_count / metrics['word_count']
    metrics['adverb_ratio'] = adverb_count / metrics['word_count']
    metrics['other_pos_ratio'] = 1 - metrics['noun_ratio'] - metrics['pronoun_ratio'] - metrics['verb_ratio'] \
                                   - metrics['adjective_ratio'] - metrics['adverb_ratio']
  else:
    metrics['noun_ratio'] = 0
    metrics['pronoun_ratio'] = 0
    metrics['verb_ratio'] = 0
    metrics['adjective_ratio'] = 0
    metrics['adverb_ratio'] = 0
    metrics['other_pos_ratio'] = 0

  # count number of modals
  modal_count = data['parts_of_speech'].count('MD')
  if metrics['word_count']:
    metrics['modal_ratio'] = modal_count / metrics['word_count']
  else:
    metrics['modal_ratio'] = 0

  # find and count nominalizations, weak verbs, entity substitutes, and filler words
  data['nominalizations'] = [False if is_word(token) else None for token in doc]
  data['weak_verbs'] = [False if is_word(token) else None for token in doc]
  data['entity_substitutions'] = [False if is_word(token) else None for token in doc]
  data['filler_words'] = [False if is_word(token) else None for token in doc]
  for idx_word, word in enumerate(words):
    idx = word2token_map[idx_word]
    data['nominalizations'][idx] = (data['numbers_of_characters'][idx] > 7) and (data['pos_high_level'][idx] != 'PROPN')\
                                    and (nominalization_re.search(word) is not None)
    data['weak_verbs'][idx] = (data['pos_high_level'][idx] == 'VERB') and (data['base_lemmas'][idx] in dict_weak_verbs)
    if data['weak_verbs'][idx] and data['auxiliary_verbs'][idx]:
      data['weak_verbs'][idx] = False
    data['entity_substitutions'][idx] = (word in dict_entity_substitutions) and (not data['values'][idx].isupper() or (word == 'i'))
  #  here
    if word in ['this', 'that']:
      if (idx > 0) and (data['parts_of_speech'][idx-1][:2] in ['NN', 'PR']):
        data['entity_substitutions'][idx] = False
      if (idx < len(doc)) and ((data['parts_of_speech'][idx+1][:2] in ['NN', 'PR', 'WP', 'JJ', 'DT', 'WD', 'WP'])
                          or (doc[idx+1].lower_ in ['there', 'that', 'this', 'here'])):
        data['entity_substitutions'][idx] = False
    data['filler_words'][idx] = (word in dict_fillers)
  if (noun_count + pronoun_nonpossesive_count) > 0:
    metrics['nominalization_ratio'] = data['nominalizations'].count(True) / (noun_count + pronoun_nonpossesive_count)
    metrics['entity_substitution_ratio'] = data['entity_substitutions'].count(True) / (noun_count + pronoun_nonpossesive_count)
  else:
    metrics['nominalization_ratio'] = 0
    metrics['entity_substitution_ratio'] = 0
  if verb_count > 0:
    metrics['weak_verb_ratio'] = data['weak_verbs'].count(True) / verb_count
  else:
    metrics['weak_verb_ratio'] = 0
  if len(words) > 0:
    metrics['filler_ratio'] = data['filler_words'].count(True) / len(words)
  else:
    metrics['filler_ratio'] = 0

  # find and count negations
  data['negations'] = [False if is_word(token) else None for token in doc]
  for idx_word, word in enumerate(words):

    idx = word2token_map[idx_word]
    if word in ["not", "n't", "no", "neither", "nor", "nothing", "nobody", "nowhere", "never"]:
      data['negations'][idx] = True
    elif (word[:2] == 'un') and (word[2:] in dict_phonemes) and (data['base_lemmas'][idx] not in ['unit', 'under', 'union', 'until']):
      data['negations'][idx] = True
    elif (word[:3] == 'mis') and (word[3:] in dict_phonemes) and (data['base_lemmas'][idx] != 'miss'):
      data['negations'][idx] = True
  if metrics['sentence_count']:
    metrics['negation_ratio'] = data['negations'].count(True) / metrics['sentence_count']
  else:
    metrics['negation_ratio'] = 0

  # find and count noun clusters
  data['noun_clusters'] = [None] * len(doc)
  noun_cluster_count = 0
  noun_count_in_cluster = 0
  total_noun_count_in_cluster = 0
  noun_cluster_span = [None, None]
  for idx, token in enumerate(doc):
    if data['parts_of_speech'][idx][:2] == 'NN':
      if noun_cluster_span[0] is None:
        noun_cluster_span = [idx, idx+1]
        noun_count_in_cluster = 1
      else:
        noun_cluster_span[1] = idx+1
        noun_count_in_cluster += 1
    elif token.lower_ not in ["'s", "of"]:
      if noun_count_in_cluster >= 3:
        noun_cluster_count += 1
        data['noun_clusters'][noun_cluster_span[0]:noun_cluster_span[1]] = \
            [noun_cluster_count] * (noun_cluster_span[1] - noun_cluster_span[0])
        total_noun_count_in_cluster += noun_count_in_cluster
      noun_cluster_span = [None, None]
      noun_count_in_cluster = 0
  if noun_count > 0:
      metrics['noun_cluster_ratio'] = total_noun_count_in_cluster / noun_count
  else:
      metrics['noun_cluster_ratio'] = 0

  # find and count long noun phrases
  data['long_noun_phrases'] = [None] * len(doc)
  noun_phrase_count = 0
  long_noun_phrase_count = 0
  for noun_phrase in doc.noun_chunks:
    noun_phrase_count += 1
    phrase_len = sum([1 for token in noun_phrase if not \
                 (token.pos_ in ['PUNCT', 'DET', 'ADP', 'CCONJ', 'PART'])])
    if phrase_len >= 5:
      long_noun_phrase_count += 1
      data['long_noun_phrases'][noun_phrase.start:noun_phrase.end] = \
          [long_noun_phrase_count] * len(noun_phrase)
  if noun_phrase_count > 0:
    metrics['long_noun_phrase_ratio'] = long_noun_phrase_count / noun_phrase_count
  else:
    metrics['long_noun_phrase_ratio'] = 0

  # find and count passive voice cases
  data['passive_voice_cases'] = [None] * len(doc)
  passive_voice_count = 0
  for i in range(verb_group_count):
    verb_group_stack = [idx for idx in range(len(doc)) if data['verb_groups'][idx] == i+1]
    if data['parts_of_speech'][verb_group_stack[-1]] in ['VBN', 'VBD']:
      for j in verb_group_stack[:-1]:
        if doc[j].lower_ in ["am", "'m", "is", "'s", "are", "'re", "was", "were", "be", "been", "being"]:
          passive_voice_count += 1
          data['passive_voice_cases'][j] = passive_voice_count
          data['passive_voice_cases'][verb_group_stack[-1]] = passive_voice_count
          break
  if metrics['sentence_count']:
    metrics['passive_voice_ratio'] = passive_voice_count / metrics['sentence_count']
  else:
    metrics['passive_voice_ratio'] = 0

  # count rare words
  if len(words):
    metrics['rare_word_ratio'] = data['expected_word_frequencies'].count(0) / len(words)
  else:
    metrics['rare_word_ratio'] = 0

  # count word, bigram, and trigram frequencies
  word_freq = {}
  bigram_freq = {}
  trigram_freq = {}
  lemmas = [data['base_lemmas'][i] for i in word2token_map]
  for idx, word in enumerate(lemmas[:-2]):
    if word in word_freq:
      word_freq[word] += 1
    else:
      word_freq[word] = 1
    mask = word + '|' + lemmas[idx+1]
    if mask in bigram_freq:
      bigram_freq[mask] += 1
    else:
      bigram_freq[mask] = 1
    mask = word + '|' + lemmas[idx+1] + '|' + lemmas[idx+2]
    if mask in trigram_freq:
      trigram_freq[mask] += 1
    else:
      trigram_freq[mask] = 1
  if lemmas[-2] in word_freq:
    word_freq[lemmas[-2]] += 1
  else:
    word_freq[lemmas[-2]] = 1
  if lemmas[-1] in word_freq:
    word_freq[lemmas[-1]] += 1
  else:
    word_freq[lemmas[-1]] = 1
  mask = lemmas[-2] + '|' + lemmas[-1]
  if mask in bigram_freq:
    bigram_freq[mask] += 1
  else:
    bigram_freq[mask] = 1
  word_freq_list = []
  for word, freq in word_freq.items():
    if (freq > 1) and (word not in dict_stop_words):
      word_freq_list.append([word, freq])
  word_freq_list.sort(reverse=True, key=lambda x: x[1])
  if word_freq_list:
    metrics['word_freq'] = word_freq_list
    # => type(metrics['word_freq']) = list
  else:
    metrics['word_freq'] = []
  bigram_freq_list = []
  for mask, freq in bigram_freq.items():
    w1, w2 = mask.split('|')
    if (freq > 1) and (w1 not in dict_stop_words) and (w2 not in dict_stop_words):
      bigram_freq_list.append([[w1, w2], freq])
  bigram_freq_list.sort(reverse=True, key=lambda x: x[1])
  if bigram_freq_list:
    metrics['bigram_freq'] = bigram_freq_list
  else:
    metrics['bigram_freq'] = []
  trigram_freq_list = []
  for mask, freq in trigram_freq.items():
    w1, w2, w3 = mask.split('|')
    if (freq > 1):
      trigram_freq_list.append([[w1, w2, w3], freq])
  trigram_freq_list.sort(reverse=True, key=lambda x: x[1])
  if trigram_freq_list:
    metrics['trigram_freq'] = trigram_freq_list
  else:
    metrics['trigram_freq'] = []
  return original_text, data, metrics