Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix to writer fails with IndexError: list index out of range where timer has fired after the test has finished #55

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Fix to writer fails with IndexError: list index out of range where timer has fired after the test has finished #55

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6f7a5ef
1 - make pycharm happy
dave00galloway Apr 3, 2018
4027eba
1 - skip creating results for timer_name items that have no custom_ti…
dave00galloway Apr 4, 2018
bcfd77f
1 - mystery solved, the list of timer names includes timers that happ…
dave00galloway Apr 4, 2018
17bccc6
1 - added config option allow_overrun which defaults to True
dave00galloway Apr 4, 2018
d459b4c
1 - updated .gitignore
dave00galloway Apr 4, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,5 @@ _build
build
dist
.tox
/.idea
*.log
231 changes: 120 additions & 111 deletions multimechanize/results.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,16 @@

import time
from collections import defaultdict

import graph
import reportwriter
import reportwriterxml



def output_results(results_dir, results_file, run_time, rampup, ts_interval, user_group_configs=None, xml_reports=False):
results = Results(results_dir + results_file, run_time)
def output_results(results_dir, results_file, run_time, rampup, ts_interval, user_group_configs=None,
xml_reports=False, allow_overrun=True):
invalid_timers = []
results = Results(results_dir + results_file, run_time, allow_overrun)

report = reportwriter.Report(results_dir)

Expand Down Expand Up @@ -49,7 +51,7 @@ def output_results(results_dir, results_file, run_time, rampup, ts_interval, use
report.write_line('<tr><th>group name</th><th>threads</th><th>script name</th></tr>')
for user_group_config in user_group_configs:
report.write_line('<tr><td>%s</td><td>%d</td><td>%s</td></tr>' %
(user_group_config.name, user_group_config.num_threads, user_group_config.script_file))
(user_group_config.name, user_group_config.num_threads, user_group_config.script_file))
report.write_line('</table>')
report.write_line('</div>')

Expand All @@ -66,20 +68,21 @@ def output_results(results_dir, results_file, run_time, rampup, ts_interval, use

report.write_line('<h3>Transaction Response Summary (secs)</h3>')
report.write_line('<table>')
report.write_line('<tr><th>count</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
report.write_line('<tr><td>%i</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>' % (
results.total_transactions,
min(trans_timer_vals),
average(trans_timer_vals),
percentile(trans_timer_vals, 80),
percentile(trans_timer_vals, 90),
percentile(trans_timer_vals, 95),
max(trans_timer_vals),
standard_dev(trans_timer_vals),
))
report.write_line(
'<tr><th>count</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
report.write_line(
'<tr><td>%i</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>' % (
results.total_transactions,
min(trans_timer_vals),
average(trans_timer_vals),
percentile(trans_timer_vals, 80),
percentile(trans_timer_vals, 90),
percentile(trans_timer_vals, 95),
max(trans_timer_vals),
standard_dev(trans_timer_vals),
))
report.write_line('</table>')


# all transactions - interval details
avg_resptime_points = {} # {intervalnumber: avg_resptime}
percentile_80_resptime_points = {} # {intervalnumber: 80pct_resptime}
Expand All @@ -88,13 +91,16 @@ def output_results(results_dir, results_file, run_time, rampup, ts_interval, use
splat_series = split_series(trans_timer_points, interval_secs)
report.write_line('<h3>Interval Details (secs)</h3>')
report.write_line('<table>')
report.write_line('<tr><th>interval</th><th>count</th><th>rate</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
report.write_line(
'<tr><th>interval</th><th>count</th><th>rate</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
for i, bucket in enumerate(splat_series):
interval_start = int((i + 1) * interval_secs)
cnt = len(bucket)

if cnt == 0:
report.write_line('<tr><td>%i</td><td>0</td><td>0</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td></tr>' % (i + 1))
report.write_line(
'<tr><td>%i</td><td>0</td><td>0</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td></tr>' % (
i + 1))
else:
rate = cnt / float(interval_secs)
mn = min(bucket)
Expand All @@ -104,15 +110,17 @@ def output_results(results_dir, results_file, run_time, rampup, ts_interval, use
pct_95 = percentile(bucket, 95)
mx = max(bucket)
stdev = standard_dev(bucket)
report.write_line('<tr><td>%i</td><td>%i</td><td>%.2f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>' % (i + 1, cnt, rate, mn, avg, pct_80, pct_90, pct_95, mx, stdev))
report.write_line(
'<tr><td>%i</td><td>%i</td><td>%.2f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>' % (
i + 1, cnt, rate, mn, avg, pct_80, pct_90, pct_95, mx, stdev))

avg_resptime_points[interval_start] = avg
percentile_80_resptime_points[interval_start] = pct_80
percentile_90_resptime_points[interval_start] = pct_90

report.write_line('</table>')
graph.resp_graph(avg_resptime_points, percentile_80_resptime_points, percentile_90_resptime_points, 'All_Transactions_response_times_intervals.png', results_dir)

graph.resp_graph(avg_resptime_points, percentile_80_resptime_points, percentile_90_resptime_points,
'All_Transactions_response_times_intervals.png', results_dir)

report.write_line('<h3>Graphs</h3>')
report.write_line('<h4>Response Time: %s sec time-series</h4>' % ts_interval)
Expand All @@ -122,8 +130,6 @@ def output_results(results_dir, results_file, run_time, rampup, ts_interval, use
report.write_line('<h4>Throughput: 5 sec time-series</h4>')
report.write_line('<img src="All_Transactions_throughput.png"></img>')



# all transactions - throughput
throughput_points = {} # {intervalnumber: numberofrequests}
interval_secs = ts_interval
Expand All @@ -132,8 +138,6 @@ def output_results(results_dir, results_file, run_time, rampup, ts_interval, use
throughput_points[int((i + 1) * interval_secs)] = (len(bucket) / interval_secs)
graph.tp_graph(throughput_points, 'All_Transactions_throughput.png', results_dir)



# custom timers
for timer_name in sorted(results.uniq_timer_names):
custom_timer_vals = []
Expand All @@ -145,81 +149,89 @@ def output_results(results_dir, results_file, run_time, rampup, ts_interval, use
custom_timer_vals.append(val)
except KeyError:
pass
graph.resp_graph_raw(custom_timer_points, timer_name + '_response_times.png', results_dir)

throughput_points = {} # {intervalnumber: numberofrequests}
interval_secs = ts_interval
splat_series = split_series(custom_timer_points, interval_secs)
for i, bucket in enumerate(splat_series):
throughput_points[int((i + 1) * interval_secs)] = (len(bucket) / interval_secs)
graph.tp_graph(throughput_points, timer_name + '_throughput.png', results_dir)

report.write_line('<hr />')
report.write_line('<h2>Custom Timer: %s</h2>' % timer_name)

report.write_line('<h3>Timer Summary (secs)</h3>')

report.write_line('<table>')
report.write_line('<tr><th>count</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
report.write_line('<tr><td>%i</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>' % (
len(custom_timer_vals),
min(custom_timer_vals),
average(custom_timer_vals),
percentile(custom_timer_vals, 80),
percentile(custom_timer_vals, 90),
percentile(custom_timer_vals, 95),
max(custom_timer_vals),
standard_dev(custom_timer_vals)
))
report.write_line('</table>')


# custom timers - interval details
avg_resptime_points = {} # {intervalnumber: avg_resptime}
percentile_80_resptime_points = {} # {intervalnumber: 80pct_resptime}
percentile_90_resptime_points = {} # {intervalnumber: 90pct_resptime}
interval_secs = ts_interval
splat_series = split_series(custom_timer_points, interval_secs)
report.write_line('<h3>Interval Details (secs)</h3>')
report.write_line('<table>')
report.write_line('<tr><th>interval</th><th>count</th><th>rate</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
for i, bucket in enumerate(splat_series):
interval_start = int((i + 1) * interval_secs)
cnt = len(bucket)

if cnt == 0:
report.write_line('<tr><td>%i</td><td>0</td><td>0</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td></tr>' % (i + 1))
else:
rate = cnt / float(interval_secs)
mn = min(bucket)
avg = average(bucket)
pct_80 = percentile(bucket, 80)
pct_90 = percentile(bucket, 90)
pct_95 = percentile(bucket, 95)
mx = max(bucket)
stdev = standard_dev(bucket)

report.write_line('<tr><td>%i</td><td>%i</td><td>%.2f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>' % (i + 1, cnt, rate, mn, avg, pct_80, pct_90, pct_95, mx, stdev))

avg_resptime_points[interval_start] = avg
percentile_80_resptime_points[interval_start] = pct_80
percentile_90_resptime_points[interval_start] = pct_90
report.write_line('</table>')
graph.resp_graph(avg_resptime_points, percentile_80_resptime_points, percentile_90_resptime_points, timer_name + '_response_times_intervals.png', results_dir)


report.write_line('<h3>Graphs</h3>')
report.write_line('<h4>Response Time: %s sec time-series</h4>' % ts_interval)
report.write_line('<img src="%s_response_times_intervals.png"></img>' % timer_name)
report.write_line('<h4>Response Time: raw data (all points)</h4>')
report.write_line('<img src="%s_response_times.png"></img>' % timer_name)
report.write_line('<h4>Throughput: %s sec time-series</h4>' % ts_interval)
report.write_line('<img src="%s_throughput.png"></img>' % timer_name)


if len(custom_timer_points) > 0:

graph.resp_graph_raw(custom_timer_points, timer_name + '_response_times.png', results_dir)

throughput_points = {} # {intervalnumber: numberofrequests}
interval_secs = ts_interval
splat_series = split_series(custom_timer_points, interval_secs)
for i, bucket in enumerate(splat_series):
throughput_points[int((i + 1) * interval_secs)] = (len(bucket) / interval_secs)
graph.tp_graph(throughput_points, timer_name + '_throughput.png', results_dir)

report.write_line('<hr />')
report.write_line('<h2>Custom Timer: %s</h2>' % timer_name)

report.write_line('<h3>Timer Summary (secs)</h3>')

report.write_line('<table>')
report.write_line(
'<tr><th>count</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
report.write_line(
'<tr><td>%i</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>' % (
len(custom_timer_vals),
min(custom_timer_vals),
average(custom_timer_vals),
percentile(custom_timer_vals, 80),
percentile(custom_timer_vals, 90),
percentile(custom_timer_vals, 95),
max(custom_timer_vals),
standard_dev(custom_timer_vals)
))
report.write_line('</table>')

# custom timers - interval details
avg_resptime_points = {} # {intervalnumber: avg_resptime}
percentile_80_resptime_points = {} # {intervalnumber: 80pct_resptime}
percentile_90_resptime_points = {} # {intervalnumber: 90pct_resptime}
interval_secs = ts_interval
splat_series = split_series(custom_timer_points, interval_secs)
report.write_line('<h3>Interval Details (secs)</h3>')
report.write_line('<table>')
report.write_line(
'<tr><th>interval</th><th>count</th><th>rate</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
for i, bucket in enumerate(splat_series):
interval_start = int((i + 1) * interval_secs)
cnt = len(bucket)

if cnt == 0:
report.write_line(
'<tr><td>%i</td><td>0</td><td>0</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td></tr>' % (
i + 1))
else:
rate = cnt / float(interval_secs)
mn = min(bucket)
avg = average(bucket)
pct_80 = percentile(bucket, 80)
pct_90 = percentile(bucket, 90)
pct_95 = percentile(bucket, 95)
mx = max(bucket)
stdev = standard_dev(bucket)

report.write_line(
'<tr><td>%i</td><td>%i</td><td>%.2f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>' % (
i + 1, cnt, rate, mn, avg, pct_80, pct_90, pct_95, mx, stdev))

avg_resptime_points[interval_start] = avg
percentile_80_resptime_points[interval_start] = pct_80
percentile_90_resptime_points[interval_start] = pct_90
report.write_line('</table>')
graph.resp_graph(avg_resptime_points, percentile_80_resptime_points, percentile_90_resptime_points,
timer_name + '_response_times_intervals.png', results_dir)

report.write_line('<h3>Graphs</h3>')
report.write_line('<h4>Response Time: %s sec time-series</h4>' % ts_interval)
report.write_line('<img src="%s_response_times_intervals.png"></img>' % timer_name)
report.write_line('<h4>Response Time: raw data (all points)</h4>')
report.write_line('<img src="%s_response_times.png"></img>' % timer_name)
report.write_line('<h4>Throughput: %s sec time-series</h4>' % ts_interval)
report.write_line('<img src="%s_throughput.png"></img>' % timer_name)
else:
invalid_timers.append(timer_name)

## user group times
#for user_group_name in sorted(results.uniq_user_group_names):
# for user_group_name in sorted(results.uniq_user_group_names):
# ug_timer_vals = []
# for resp_stats in results.resp_stats_list:
# if resp_stats.user_group_name == user_group_name:
Expand All @@ -234,13 +246,18 @@ def output_results(results_dir, results_file, run_time, rampup, ts_interval, use
# print ''

report.write_line('<hr />')
if len(invalid_timers) > 0:
report.write_line('<h2>Invalid Timers</h2>')
report.write_line('<ul>')
for t in invalid_timers:
report.write_line('<li>' + t + '</li>')
report.write_line('</ul>')
report.write_closing_html()




class Results(object):
def __init__(self, results_file_name, run_time):
def __init__(self, results_file_name, run_time, allow_overrun=True):
self.allow_overrun = allow_overrun
self.results_file_name = results_file_name
self.run_time = run_time
self.total_transactions = 0
Expand All @@ -255,8 +272,6 @@ def __init__(self, results_file_name, run_time):
self.start_datetime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(self.epoch_start))
self.finish_datetime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(self.epoch_finish))



def __parse_file(self):
f = open(self.results_file_name, 'rb')
resp_stats_list = []
Expand Down Expand Up @@ -289,7 +304,8 @@ def __parse_file(self):

r = ResponseStats(request_num, elapsed_time, epoch_secs, user_group_name, trans_time, error, custom_timers)

if elapsed_time < self.run_time: # drop all times that appear after the last request was sent (incomplete interval)
# drop all times that appear after the last request was sent (incomplete interval) unless overrun allowed
if self.allow_overrun or elapsed_time < self.run_time:
resp_stats_list.append(r)

if error != '':
Expand All @@ -300,7 +316,6 @@ def __parse_file(self):
return resp_stats_list



class ResponseStats(object):
def __init__(self, request_num, elapsed_time, epoch_secs, user_group_name, trans_time, error, custom_timers):
self.request_num = request_num
Expand All @@ -312,7 +327,6 @@ def __init__(self, request_num, elapsed_time, epoch_secs, user_group_name, trans
self.custom_timers = custom_timers



def split_series(points, interval):
offset = points[0][0]
maxval = int((points[-1][0] - offset) // interval)
Expand All @@ -323,13 +337,11 @@ def split_series(points, interval):
return series



def average(seq):
avg = (float(sum(seq)) / len(seq))
return avg



def standard_dev(seq):
avg = average(seq)
sdsq = sum([(i - avg) ** 2 for i in seq])
Expand All @@ -340,14 +352,11 @@ def standard_dev(seq):
return stdev



def percentile(seq, percentile):
i = int(len(seq) * (percentile / 100.0))
seq.sort()
return seq[i]




if __name__ == '__main__':
output_results('./', 'results.csv', 60, 30, 10)
Loading