-
Notifications
You must be signed in to change notification settings - Fork 0
/
eval.py
executable file
·46 lines (38 loc) · 1.87 KB
/
eval.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
import predict
import time as tm
import numpy as np
# Perform longest common subsequence search on the (truncated) code and gold code
def lcs( str1, str2, p, q ):
if p == 0 or q == 0:
return 0
elif str1[p-1] == str2[q-1]:
return 1 + lcs( str1, str2, p - 1, q - 1 )
else:
return max( lcs( str1, str2, p - 1, q ), lcs( str1, str2, p, q - 1 ) )
def getLCS( code, goldCode ):
return lcs( code, goldCode, min( len( code ), 4 ), len( goldCode ) )
# If there are 100 test points and the prediction code returns 110 predictions
# then we only consider the first 100 and discard the last 10 predictions. On
# the other hand, if the code returns only 90 predictions, then we assume that
# these were predictions on the first 90 test points and evaluate accordingly
def getCodeLengthMatch( numChars, goldNumChars ):
minLen = min( len( codes ), len( goldCodes ) )
return np.count_nonzero( numChars[0:minLen] == goldNumChars[0:minLen] )
def getCodeMatchScore( codes, goldCodes ):
totScore = 0
for i in range( min( len( codes ), len( goldCodes ) ) ):
totScore += getLCS( codes[i], goldCodes[i] ) / len( goldCodes[i] )
return totScore
numTest = 2000
filepaths = [ "test/image%d.png" % i for i in range( numTest ) ]
file = open( "test/codes.txt", "r" )
goldCodes = file.read().splitlines()
file.close()
goldNumChars = np.array( [ len( goldCodes[i] ) for i in range( len( goldCodes ) ) ] )
# Get recommendations from predict.py and time the thing
tic = tm.perf_counter()
(numChars, codes) = predict.decaptcha( filepaths )
toc = tm.perf_counter()
print( "Total time taken is %.6f seconds " % (toc - tic) )
print( "Fraction of code lengths that match is %.6f" % (getCodeLengthMatch( numChars, goldNumChars ) / numTest) )
print( "Code match score is %.6f" % (getCodeMatchScore( codes, goldCodes ) / numTest) )