Subtlex OOV

[OmerShubi]

text-metrics/eyeutils/utils.py

Lines 127 to 140 in 44ee12d

	# TODO subtlex freq should be 'inf' if missing, not zero?
	subtlex_freqs = []
	for word in words:
	tokens = tokenize(word, lang='en')
	one_over_result = 0.0
	try:
	for token in tokens:
	one_over_result += 1.0 / subtlex.loc[token, 'Frequency']
	except KeyError:
	subtlex_freq = 0
	else:
	subtlex_freq = 1.0 / one_over_result if one_over_result != 0 else 0
	subtlex_freqs.append(subtlex_freq)
	frequencies['subtlex_Frequency'] = subtlex_freqs