Scenario should fail if the actual recovery time exceeds the threshold set

[chaitanyaenr]

Here's the telemetry data from one of the runs:

"telemetry": {
        "scenarios": [
            {
                "start_timestamp": 1720971022,
                "end_timestamp": 1720971089,
                "scenario": "scenarios/pod_scenario.yaml",
                "exit_status": 0,
                "parameters_base64": "",
                "parameters": [
                    {
                        "config": {
                            "kill": 1,
                            "krkn_pod_recovery_time": 100,
                            "label_selector": "prometheus=k8s",
                            "namespace_pattern": "openshift-monitoring",
                            "timeout": 180
                        },
                        "id": "kill-pods"
                    }
                ],
                "affected_pods": {
                    "recovered": [
                        {
                            "pod_name": "prometheus-k8s-1",
                            "namespace": "openshift-monitoring",
                            "total_recovery_time": 122.3113272190094,
                            "pod_readiness_time": 60.979615211486816,
                            "pod_rescheduling_time": 61.33171200752258
                        }
                    ],
                    "unrecovered": [],
                    "error": null
                }
            }
        ],