Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tidy eval metrics, make optional. #206

Merged
merged 8 commits into from
Jul 15, 2024
Merged

Tidy eval metrics, make optional. #206

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
babd87d
Adds extra video stream overlays, also adds laptime best model metric
mattcamp May 29, 2024
ac9ce08
fix bool flag for extra kvs overlay
mattcamp May 29, 2024
91e540f
remove debug statements
mattcamp May 29, 2024
49b5c6a
move kvs overlay toggle to rospy param
mattcamp Jun 1, 2024
b3fb865
fix rospy param
mattcamp Jun 1, 2024
3667de3
tidy eval metrics, make optional
mattcamp Jul 1, 2024
310476f
Merge branch 'dev' into matt/extra_stream_overlay
larsll Jul 13, 2024
f897aae
fix overlay rounding
mattcamp Jul 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions bundle/markov/metrics/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,9 @@ class Mp4VideoMetrics(Enum):
CRASH_COUNTER = 'crash_counter'
THROTTLE = 'throttle'
STEERING = 'steering'
SPEED = 'speed'
BEST_LAP_TIME = 'best_lap_time'
LAST_LAP_TIME = 'last_lap_time'
TOTAL_EVALUATION_TIME = 'total_evaluation_time'
DONE = 'done'
X = 'x'
Expand All @@ -132,3 +134,4 @@ class BestModelMetricType(Enum):
"""This enum is used to determine the metric to use when selecting best model"""
PROGRESS = 'progress'
REWARD = 'reward'
LAPTIME = 'laptime'
86 changes: 70 additions & 16 deletions bundle/markov/metrics/s3_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,12 @@ def __init__(self, agent_name, s3_dict_metrics, deepracer_checkpoint_json, ckpnt
self._progress_ = 0.0
self._episode_status = ''
self._metrics_ = list()
self._agent_xy = list()
self._prev_step_time = time.time()
self._is_eval_ = True
self._eval_trials_ = 0
self._best_lap_time = float('inf')
self._last_lap_time = float('inf')
self._checkpoint_state_ = CheckpointStateFile(ckpnt_dir)
self._use_model_picker = use_model_picker
self._eval_stats_dict_ = {'chkpnt_name': None,
Expand Down Expand Up @@ -151,6 +155,11 @@ def append_episode_metrics(self):
training_metric['completion_percentage'] = int(self._progress_)
training_metric['episode_status'] = EpisodeStatus.get_episode_status_label(self._episode_status)
self._metrics_.append(training_metric)

if self._episode_status == EpisodeStatus.EPISODE_COMPLETE.value:
self._best_lap_time = min(training_metric['elapsed_time_in_milliseconds'], self._best_lap_time)
self._last_lap_time = training_metric['elapsed_time_in_milliseconds']

if self._telegraf_client:
self._telegraf_client.metric('dr_training_episodes',
{'reward':training_metric['reward_score'],
Expand All @@ -171,6 +180,11 @@ def upload_episode_metrics(self):
if self._is_eval_:
if self._best_model_metric_type == BestModelMetricType.REWARD:
self._current_eval_best_model_metric_list_.append(self._episode_reward_)
elif self._best_model_metric_type == BestModelMetricType.LAPTIME:
if self._progress_ >= 100:
self._current_eval_best_model_metric_list_.append(int(round((self._current_sim_time - self._start_time_) * 1000)))
else:
self._current_eval_best_model_metric_list_.append(999999)
else:
self._current_eval_best_model_metric_list_.append(self._progress_)

Expand Down Expand Up @@ -204,18 +218,32 @@ def update(self, data):
self._current_eval_best_model_metric_list_.clear()

time_stamp = self._current_sim_time
if self._eval_stats_dict_['avg_eval_metric'] is None or \
mean_metric >= self._eval_stats_dict_['avg_eval_metric']:
msg_format = '[BestModelSelection] current {0} mean: {1} >= best {0} mean: {2}'
LOGGER.info(msg_format.format(self._best_model_metric_type.value,
mean_metric,
self._eval_stats_dict_['avg_eval_metric']))
msg_format = '[BestModelSelection] Updating the best checkpoint to "{}" from "{}".'
LOGGER.info(msg_format.format(self._eval_stats_dict_['chkpnt_name'], self._best_chkpnt_stats['name']))
self._eval_stats_dict_['avg_eval_metric'] = mean_metric
self._best_chkpnt_stats = {'name': self._eval_stats_dict_['chkpnt_name'],
'avg_eval_metric': mean_metric,
'time_stamp': time_stamp}
if self._best_model_metric_type == BestModelMetricType.LAPTIME:
if self._eval_stats_dict_['avg_eval_metric'] is None or \
mean_metric <= self._eval_stats_dict_['avg_eval_metric']:
msg_format = '[BestModelSelection] current {0} mean: {1} >= best {0} mean: {2}'
LOGGER.info(msg_format.format(self._best_model_metric_type.value,
mean_metric,
self._eval_stats_dict_['avg_eval_metric']))
msg_format = '[BestModelSelection] Updating the best checkpoint to "{}" from "{}".'
LOGGER.info(msg_format.format(self._eval_stats_dict_['chkpnt_name'], self._best_chkpnt_stats['name']))
self._eval_stats_dict_['avg_eval_metric'] = mean_metric
self._best_chkpnt_stats = {'name': self._eval_stats_dict_['chkpnt_name'],
'avg_eval_metric': mean_metric,
'time_stamp': time_stamp}
else:
if self._eval_stats_dict_['avg_eval_metric'] is None or \
mean_metric >= self._eval_stats_dict_['avg_eval_metric']:
msg_format = '[BestModelSelection] current {0} mean: {1} >= best {0} mean: {2}'
LOGGER.info(msg_format.format(self._best_model_metric_type.value,
mean_metric,
self._eval_stats_dict_['avg_eval_metric']))
msg_format = '[BestModelSelection] Updating the best checkpoint to "{}" from "{}".'
LOGGER.info(msg_format.format(self._eval_stats_dict_['chkpnt_name'], self._best_chkpnt_stats['name']))
self._eval_stats_dict_['avg_eval_metric'] = mean_metric
self._best_chkpnt_stats = {'name': self._eval_stats_dict_['chkpnt_name'],
'avg_eval_metric': mean_metric,
'time_stamp': time_stamp}
last_chkpnt_stats = {'name': self._eval_stats_dict_['chkpnt_name'],
'avg_eval_metric': mean_metric,
'time_stamp': time_stamp}
Expand All @@ -230,16 +258,33 @@ def update(self, data):
self._eval_stats_dict_['chkpnt_name'] = self._checkpoint_state_.read().name

def update_mp4_video_metrics(self, metrics):
actual_speed = 0
cur_time = self._current_sim_time
agent_x, agent_y = metrics[StepMetrics.X.value], metrics[StepMetrics.Y.value]
if self._agent_xy:
# Speed = Distance/Time
delta_time = cur_time - self._prev_step_time
actual_speed = 0
if delta_time:
actual_speed = math.sqrt((self._agent_xy[0] - agent_x) ** 2 +
(self._agent_xy[1] - agent_y) ** 2) / delta_time
self._agent_xy = [agent_x, agent_y]
self._prev_step_time = cur_time

# agent_x, agent_y = metrics[StepMetrics.X.value], metrics[StepMetrics.Y.value]
self._video_metrics[Mp4VideoMetrics.LAP_COUNTER.value] = 0
self._video_metrics[Mp4VideoMetrics.COMPLETION_PERCENTAGE.value] = self._progress_
# For continuous race, MP4 video will display the total reset counter for the entire race
# For non-continuous race, MP4 video will display reset counter per lap
self._video_metrics[Mp4VideoMetrics.RESET_COUNTER.value] = 0

self._video_metrics[Mp4VideoMetrics.THROTTLE.value] = 0
self._video_metrics[Mp4VideoMetrics.STEERING.value] = 0
self._video_metrics[Mp4VideoMetrics.BEST_LAP_TIME.value] = 0
# self._video_metrics[Mp4VideoMetrics.THROTTLE.value] = 0
# self._video_metrics[Mp4VideoMetrics.STEERING.value] = 0
self._video_metrics[Mp4VideoMetrics.SPEED.value] = actual_speed
self._video_metrics[Mp4VideoMetrics.THROTTLE.value] = metrics[StepMetrics.THROTTLE.value]
self._video_metrics[Mp4VideoMetrics.STEERING.value] = metrics[StepMetrics.STEER.value]
self._video_metrics[Mp4VideoMetrics.BEST_LAP_TIME.value] = self._best_lap_time
self._video_metrics[Mp4VideoMetrics.LAST_LAP_TIME.value] = self._last_lap_time
self._video_metrics[Mp4VideoMetrics.TOTAL_EVALUATION_TIME.value] = 0
self._video_metrics[Mp4VideoMetrics.DONE.value] = metrics[StepMetrics.DONE.value]
self._video_metrics[Mp4VideoMetrics.X.value] = agent_x
Expand All @@ -258,9 +303,11 @@ def _handle_get_video_metrics(self, req):
return VideoMetricsSrvResponse(self._video_metrics[Mp4VideoMetrics.LAP_COUNTER.value],
self._video_metrics[Mp4VideoMetrics.COMPLETION_PERCENTAGE.value],
self._video_metrics[Mp4VideoMetrics.RESET_COUNTER.value],
self._video_metrics[Mp4VideoMetrics.SPEED.value],
self._video_metrics[Mp4VideoMetrics.THROTTLE.value],
self._video_metrics[Mp4VideoMetrics.STEERING.value],
self._video_metrics[Mp4VideoMetrics.BEST_LAP_TIME.value],
self._video_metrics[Mp4VideoMetrics.LAST_LAP_TIME.value],
self._video_metrics[Mp4VideoMetrics.TOTAL_EVALUATION_TIME.value],
self._video_metrics[Mp4VideoMetrics.DONE.value],
self._video_metrics[Mp4VideoMetrics.X.value],
Expand Down Expand Up @@ -317,6 +364,7 @@ def __init__(self, agent_name, s3_dict_metrics, is_continuous, pause_time_before
EpisodeStatus.IMMOBILIZED.value: 0,
EpisodeStatus.REVERSED.value: 0}
self._best_lap_time = float('inf')
self._last_lap_time = float('inf')
self._total_evaluation_time = 0
self._video_metrics = Mp4VideoMetrics.get_empty_dict()
self._reset_count_sum = 0
Expand Down Expand Up @@ -406,6 +454,7 @@ def append_episode_metrics(self, is_complete=True):
self._number_of_trials_ += 1
self._best_lap_time = min(eval_metric['elapsed_time_in_milliseconds'], self._best_lap_time)
self._total_evaluation_time += eval_metric['elapsed_time_in_milliseconds']
self._last_lap_time = eval_metric['elapsed_time_in_milliseconds']
eval_metric['trial'] = int(self._number_of_trials_)
self._metrics_.append(eval_metric)
if self._telegraf_client:
Expand Down Expand Up @@ -454,9 +503,11 @@ def update_mp4_video_metrics(self, metrics):
self.reset_count_dict[EpisodeStatus.REVERSED.value] + \
(self._reset_count_sum if self._is_continuous else 0)

self._video_metrics[Mp4VideoMetrics.THROTTLE.value] = actual_speed
self._video_metrics[Mp4VideoMetrics.SPEED.value] = actual_speed
self._video_metrics[Mp4VideoMetrics.THROTTLE.value] = metrics[StepMetrics.THROTTLE.value]
self._video_metrics[Mp4VideoMetrics.STEERING.value] = metrics[StepMetrics.STEER.value]
self._video_metrics[Mp4VideoMetrics.BEST_LAP_TIME.value] = self._best_lap_time
self._video_metrics[Mp4VideoMetrics.LAST_LAP_TIME.value] = self._last_lap_time
self._video_metrics[Mp4VideoMetrics.TOTAL_EVALUATION_TIME.value] = self._total_evaluation_time +\
int(round((self._current_sim_time - self._start_time_) * 1000))
self._video_metrics[Mp4VideoMetrics.DONE.value] = metrics[StepMetrics.DONE.value]
Expand Down Expand Up @@ -492,9 +543,11 @@ def _handle_get_video_metrics(self, req):
return VideoMetricsSrvResponse(self._video_metrics[Mp4VideoMetrics.LAP_COUNTER.value],
self._video_metrics[Mp4VideoMetrics.COMPLETION_PERCENTAGE.value],
self._video_metrics[Mp4VideoMetrics.RESET_COUNTER.value],
self._video_metrics[Mp4VideoMetrics.SPEED.value],
self._video_metrics[Mp4VideoMetrics.THROTTLE.value],
self._video_metrics[Mp4VideoMetrics.STEERING.value],
self._video_metrics[Mp4VideoMetrics.BEST_LAP_TIME.value],
self._video_metrics[Mp4VideoMetrics.LAST_LAP_TIME.value],
self._video_metrics[Mp4VideoMetrics.TOTAL_EVALUATION_TIME.value],
self._video_metrics[Mp4VideoMetrics.DONE.value],
self._video_metrics[Mp4VideoMetrics.X.value],
Expand All @@ -519,6 +572,7 @@ def clear(self):
EpisodeStatus.IMMOBILIZED.value: 0,
EpisodeStatus.REVERSED.value: 0}
self._best_lap_time = float('inf')
self._last_lap_time = float('inf')
self._total_evaluation_time = 0
self._video_metrics = Mp4VideoMetrics.get_empty_dict()
self._reset_count_sum = 0
Expand Down
21 changes: 20 additions & 1 deletion bundle/src/deepracer_simulation_environment/scripts/mp4_saving/multi_agent_image_editing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import logging
import rospy
import cv2
import os

from markov.log_handler.logger import Logger
from markov.reset.constants import RaceType
Expand Down Expand Up @@ -105,8 +106,26 @@ def _edit_major_cv_image(self, major_cv_image, mp4_video_metrics_info):
loc=(loc_x, loc_y), font=self.amazon_ember_light_18px,
font_color=RaceCarColorToRGB.White.value,
font_shadow_color=RaceCarColorToRGB.Black.value)

if rospy.get_param('ENABLE_EXTRA_KVS_OVERLAY', 'False').lower() in ('true'):
# Steering Angle
loc_y += 25
steering_text = "Steering | {}".format(mp4_video_metrics_info[i].steering)
major_cv_image = utils.write_text_on_image(image=major_cv_image, text=steering_text,
loc=(loc_x, loc_y), font=self.amazon_ember_light_18px,
font_color=RaceCarColorToRGB.White.value,
font_shadow_color=RaceCarColorToRGB.Black.value)

# Throttle
loc_y += 25
steering_text = "Throttle | {}".format(round(mp4_video_metrics_info[i].throttle,2))
major_cv_image = utils.write_text_on_image(image=major_cv_image, text=steering_text,
loc=(loc_x, loc_y), font=self.amazon_ember_light_18px,
font_color=RaceCarColorToRGB.White.value,
font_shadow_color=RaceCarColorToRGB.Black.value)

if self.racecar_name == racecar_info['name']:
agents_speed = mp4_video_metrics_info[i].throttle
agents_speed = mp4_video_metrics_info[i].speed
# The race is complete when total lap is same as current lap and done flag is set
agent_done = agent_done or (mp4_video_metrics_info[i].done and (current_lap == int(self._total_laps)))

Expand Down
52 changes: 43 additions & 9 deletions bundle/src/deepracer_simulation_environment/scripts/mp4_saving/single_agent_image_editing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import logging
import rospy
import cv2
import os

from markov.log_handler.logger import Logger
from markov.utils import get_racecar_idx
Expand Down Expand Up @@ -103,22 +104,55 @@ def _edit_major_cv_image(self, major_cv_image, mp4_video_metrics_info):
loc=(loc_x, loc_y), font=self.amazon_ember_light_18px,
font_color=RaceCarColorToRGB.White.value,
font_shadow_color=RaceCarColorToRGB.Black.value)
if rospy.get_param('ENABLE_EXTRA_KVS_OVERLAY', 'False').lower() in ('true'):
loc_y += 25
best_lap_time = mp4_video_metrics_info[self.racecar_index].best_lap_time
# The initial default best_lap_time from s3_metrics.py is inf
# If the ros service in s3_metrics.py has not come up yet, best_lap_time is 0
best_lap_time = utils.milliseconds_to_timeformat(
datetime.timedelta(milliseconds=best_lap_time)) \
if best_lap_time != float("inf") and best_lap_time != 0 else "--:--.---"

best_lap_time_text = "Best lap | {}".format(best_lap_time)
major_cv_image = utils.write_text_on_image(image=major_cv_image, text=best_lap_time_text,
loc=(loc_x, loc_y), font=self.amazon_ember_light_18px,
font_color=RaceCarColorToRGB.White.value,
font_shadow_color=RaceCarColorToRGB.Black.value)

# Reset counter
loc_y += 25
reset_counter_text = "Reset | {}".format(mp4_video_metrics_info[self.racecar_index].reset_counter)
major_cv_image = utils.write_text_on_image(image=major_cv_image, text=reset_counter_text,
loc=(loc_x, loc_y), font=self.amazon_ember_light_18px,
font_color=RaceCarColorToRGB.White.value,
font_shadow_color=RaceCarColorToRGB.Black.value)
# Speed
loc_x, loc_y = XYPixelLoc.SPEED_EVAL_LOC.value
if self.is_league_leaderboard:
loc_x, loc_y = XYPixelLoc.SPEED_LEADERBOARD_LOC.value
speed_text = "{} m/s".format(utils.get_speed_formatted_str(mp4_video_metrics_info[self.racecar_index].throttle))
major_cv_image = utils.write_text_on_image(image=major_cv_image, text=speed_text,
loc=(loc_x, loc_y), font=self.amazon_ember_light_20px,
font_color=RaceCarColorToRGB.White.value,
font_shadow_color=RaceCarColorToRGB.Black.value)
if rospy.get_param('ENABLE_EXTRA_KVS_OVERLAY', 'False').lower() in ('true'):
# Steering Angle
loc_y += 25
steering_text = "Steering | {}".format(mp4_video_metrics_info[self.racecar_index].steering)
major_cv_image = utils.write_text_on_image(image=major_cv_image, text=steering_text,
loc=(loc_x, loc_y), font=self.amazon_ember_light_18px,
font_color=RaceCarColorToRGB.White.value,
font_shadow_color=RaceCarColorToRGB.Black.value)

# Throttle
loc_y += 25
steering_text = "Throttle | {}".format(round(mp4_video_metrics_info[self.racecar_index].throttle,2))
major_cv_image = utils.write_text_on_image(image=major_cv_image, text=steering_text,
loc=(loc_x, loc_y), font=self.amazon_ember_light_18px,
font_color=RaceCarColorToRGB.White.value,
font_shadow_color=RaceCarColorToRGB.Black.value)

# Speed
loc_x, loc_y = XYPixelLoc.SPEED_EVAL_LOC.value
if self.is_league_leaderboard:
loc_x, loc_y = XYPixelLoc.SPEED_LEADERBOARD_LOC.value
speed_text = "{} m/s".format(utils.get_speed_formatted_str(mp4_video_metrics_info[self.racecar_index].speed))
major_cv_image = utils.write_text_on_image(image=major_cv_image, text=speed_text,
loc=(loc_x, loc_y), font=self.amazon_ember_light_20px,
font_color=RaceCarColorToRGB.White.value,
font_shadow_color=RaceCarColorToRGB.Black.value)

# Leaderboard name
if self.is_league_leaderboard:
loc_x, loc_y = XYPixelLoc.LEADERBOARD_NAME_LOC.value
Expand Down
Loading