From e2394c6e27122610c348072d743f1a40beb5e0cf Mon Sep 17 00:00:00 2001 From: nanitebased Date: Thu, 12 May 2022 12:45:30 +0200 Subject: [PATCH 1/3] Implement Robot Framework test execution, report gathering --- Executor/Tasks/Run/__init__.py | 1 + Executor/Tasks/Run/robot_framework.py | 56 +++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 Executor/Tasks/Run/robot_framework.py diff --git a/Executor/Tasks/Run/__init__.py b/Executor/Tasks/Run/__init__.py index fed00fc..402301e 100644 --- a/Executor/Tasks/Run/__init__.py +++ b/Executor/Tasks/Run/__init__.py @@ -13,3 +13,4 @@ from .rest_api import RestApi from .upgrade_verdict import UpgradeVerdict from .evaluate import Evaluate +from .robot_framework import RobotFramework diff --git a/Executor/Tasks/Run/robot_framework.py b/Executor/Tasks/Run/robot_framework.py new file mode 100644 index 0000000..550808c --- /dev/null +++ b/Executor/Tasks/Run/robot_framework.py @@ -0,0 +1,56 @@ +from Task import Task +from Helper import Cli, Level +from os.path import abspath, join, exists +from datetime import datetime, timezone + + +class RobotFramework(Task): + def __init__(self, logMethod, parent, params): + super().__init__("Robot Framework", parent, params, logMethod, None) + self.paramRules = { + 'Executable': (None, True), + 'Paths': (None, True), + 'CWD': (None, True), + 'GatherResults': (True, False), + 'Identifier': (None, False) + } + + def Run(self): + gatherResults = self.params['GatherResults'] + paths = self.params['Paths'] + if isinstance(paths, str): + paths = [paths] + identifier = self.params['Identifier'] # Save to different folders, in case we run multiple instances in a row + identifier = identifier if identifier is not None else f'RobotFw{datetime.now(timezone.utc).strftime("%H%M%S")}' + tempFolder = join(abspath(self.parent.TempFolder), identifier) + + parameters = [self.params['Executable']] + if gatherResults: + parameters.extend(['--outputdir', tempFolder]) + parameters.extend(paths) + + self.Log(Level.INFO, "Executing Robot Framework tests") + cli = Cli(parameters, self.params['CWD'], self.Log) + cli.Execute() + self.Log(Level.INFO, "Robot Framework tests finished") + + if gatherResults: + self.Log(Level.INFO, "Gathering test report files") + found = [] + for kind, path in [('Output', join(tempFolder, 'output.xml')), + ('Log', join(tempFolder, 'log.html')), + ('Report', join(tempFolder, 'report.html'))]: + if exists(path): + found.append(path) + else: + self.Log(Level.WARNING, f'Could not retrieve {kind} file ({path})') + if len(found) > 0: + from Helper import Compress + path = join(abspath(self.parent.TempFolder), f"{identifier}.zip") + Compress.Zip(found, path, flat=True) + self.parent.GeneratedFiles.append(path) + self.Log(Level.INFO, f"Report files compressed to '{identifier}.zip'") + else: + self.Log(Level.WARNING, "No report files found, skipping zip creation.") + else: + self.Log(Level.INFO, "'GatherResults' disabled: Reports will be generated in the CWD folder") From 3e3edc77ad8c0084170f1f99b051f0190b26279d Mon Sep 17 00:00:00 2001 From: nanitebased Date: Fri, 13 May 2022 11:22:31 +0200 Subject: [PATCH 2/3] Improve exception handling, manage step verdict --- Executor/Tasks/Run/robot_framework.py | 82 ++++++++++++++++++++------- 1 file changed, 60 insertions(+), 22 deletions(-) diff --git a/Executor/Tasks/Run/robot_framework.py b/Executor/Tasks/Run/robot_framework.py index 550808c..6491963 100644 --- a/Executor/Tasks/Run/robot_framework.py +++ b/Executor/Tasks/Run/robot_framework.py @@ -2,6 +2,7 @@ from Helper import Cli, Level from os.path import abspath, join, exists from datetime import datetime, timezone +import xml.etree.ElementTree as ET class RobotFramework(Task): @@ -12,10 +13,14 @@ def __init__(self, logMethod, parent, params): 'Paths': (None, True), 'CWD': (None, True), 'GatherResults': (True, False), - 'Identifier': (None, False) + 'Identifier': (None, False), + 'VerdictOnPass': ("Pass", False), + 'VerdictOnFail': ("Fail", False) } def Run(self): + from Executor import Verdict + gatherResults = self.params['GatherResults'] paths = self.params['Paths'] if isinstance(paths, str): @@ -23,34 +28,67 @@ def Run(self): identifier = self.params['Identifier'] # Save to different folders, in case we run multiple instances in a row identifier = identifier if identifier is not None else f'RobotFw{datetime.now(timezone.utc).strftime("%H%M%S")}' tempFolder = join(abspath(self.parent.TempFolder), identifier) + onPass = self.GetVerdictFromName(self.params["VerdictOnPass"]) + onFail = self.GetVerdictFromName(self.params["VerdictOnFail"]) - parameters = [self.params['Executable']] + parameters = [self.params['Executable'], '--xunit', 'xunit.xml'] if gatherResults: parameters.extend(['--outputdir', tempFolder]) parameters.extend(paths) - self.Log(Level.INFO, "Executing Robot Framework tests") - cli = Cli(parameters, self.params['CWD'], self.Log) - cli.Execute() - self.Log(Level.INFO, "Robot Framework tests finished") + try: + self.Log(Level.INFO, "Executing Robot Framework tests") + cli = Cli(parameters, self.params['CWD'], self.Log) + cli.Execute() + self.Log(Level.INFO, "Robot Framework tests finished") + except Exception as e: + self.SetVerdictOnError() + raise RuntimeError(f"Exception while executing Robot Framework: {e}") from e + + fullXml = join(tempFolder, 'output.xml') + if onPass != Verdict.NotSet or onFail != Verdict.NotSet: + try: + self.Log(Level.INFO, "Generating verdict from test results") + xml = ET.parse(fullXml) + total = xml.getroot().find("./statistics/total/stat") + if total is not None: + success = total.attrib['pass'] + fail = total.attrib['fail'] + skip = total.attrib['skip'] + self.Verdict = onPass if fail == 0 else onFail + self.Log(Level.INFO, + f"Verdict set to {self.Verdict.name} (pass: {success}, fail: {fail}, skip: {skip})") + else: + raise RuntimeError("Could not find total statistics in output.xml") + except Exception as e: + self.SetVerdictOnError() + raise RuntimeError(f"Exception while generating verdict: {e}") from e + + else: + self.Log(Level.INFO, "Skipping Verdict generation") if gatherResults: - self.Log(Level.INFO, "Gathering test report files") - found = [] - for kind, path in [('Output', join(tempFolder, 'output.xml')), - ('Log', join(tempFolder, 'log.html')), - ('Report', join(tempFolder, 'report.html'))]: - if exists(path): - found.append(path) + try: + self.Log(Level.INFO, "Gathering test report files") + found = [] + for kind, path in [('Output', fullXml), + ('Log', join(tempFolder, 'log.html')), + ('Report', join(tempFolder, 'report.html')), + ('xUnit compatible', join(tempFolder, 'xunit.xml'))]: + if exists(path): + found.append(path) + else: + self.Log(Level.WARNING, f'Could not retrieve {kind} file ({path})') + if len(found) > 0: + from Helper import Compress + path = join(abspath(self.parent.TempFolder), f"{identifier}.zip") + Compress.Zip(found, path, flat=True) + self.parent.GeneratedFiles.append(path) + self.Log(Level.INFO, f"Report files compressed to '{identifier}.zip'") else: - self.Log(Level.WARNING, f'Could not retrieve {kind} file ({path})') - if len(found) > 0: - from Helper import Compress - path = join(abspath(self.parent.TempFolder), f"{identifier}.zip") - Compress.Zip(found, path, flat=True) - self.parent.GeneratedFiles.append(path) - self.Log(Level.INFO, f"Report files compressed to '{identifier}.zip'") - else: - self.Log(Level.WARNING, "No report files found, skipping zip creation.") + self.Log(Level.WARNING, "No report files found, skipping zip creation.") + except Exception as e: + self.SetVerdictOnError() + raise RuntimeError(f"Exception while gathering generated files: {e}") from e else: self.Log(Level.INFO, "'GatherResults' disabled: Reports will be generated in the CWD folder") From 532e57186d5a555cbbe9e67103001ea8d5eddab4 Mon Sep 17 00:00:00 2001 From: nanitebased Date: Fri, 13 May 2022 11:49:33 +0200 Subject: [PATCH 3/3] Fix stats retrieval, update documentation --- CHANGELOG.md | 4 ++++ Executor/Tasks/Run/robot_framework.py | 6 +++--- README.md | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e82b4f1..ae0ab2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +**13/05/2022** [Version 3.4.0] + + - Add Robot Framework task + **04/05/2022** [Version 3.3.1] - Fix SliceId expansion, exception on SingleSliceCreationTime diff --git a/Executor/Tasks/Run/robot_framework.py b/Executor/Tasks/Run/robot_framework.py index 6491963..a3171b0 100644 --- a/Executor/Tasks/Run/robot_framework.py +++ b/Executor/Tasks/Run/robot_framework.py @@ -52,9 +52,9 @@ def Run(self): xml = ET.parse(fullXml) total = xml.getroot().find("./statistics/total/stat") if total is not None: - success = total.attrib['pass'] - fail = total.attrib['fail'] - skip = total.attrib['skip'] + success = int(total.attrib['pass']) + fail = int(total.attrib['fail']) + skip = int(total.attrib['skip']) self.Verdict = onPass if fail == 0 else onFail self.Log(Level.INFO, f"Verdict set to {self.Verdict.name} (pass: {success}, fail: {fail}, skip: {skip})") diff --git a/README.md b/README.md index e4c033a..6aab037 100644 --- a/README.md +++ b/README.md @@ -577,6 +577,23 @@ success response (2xx). Set to `None` to disable the check. - `Timeout`: Maximum time in seconds to wait for a response - `Headers`: Additional headers to add to the request +### Run.RobotFramework +Execute one or more test suites using an external Robot Framework instance. It is recommended to store and configure +Robot Framework in a dedicated virtualenv, where all the required dependencies (for example `robotframework-requests`) +are also installed. Configuration values: +- `Executable`: Absolute path to the Robot Framework executable. On a pre-configured virtualenv this file is usually +`/bin/robot` or `/Scripts/robot.exe` +- `Paths`: Either a single path (string) or a list of paths, each with the location of one of the test suites to run. +- `CWD`: Working directory, usually the root folder where the test suites are stored. If `GatherResults` is set to +`False` the generated report files will be left in this folder. +- `GatherResults`: Whether or not to store the generated files along with other files created by the experiment. +These reports will be compressed in a single zip file identified by the `Identifier` parameter. `True` by default. +- `Identifier`: Name used to identify a particular Robot Framework execution, in order to avoid overwriting results +for TestCases that include multiple invocations. If not set, it will be automatically generated from the time as +`RobotFwHHMMSS`, where HHMMSS corresponds to the hour, minutes and seconds in UTC. +- `VerdictOnPass`: Verdict to set if all tests are completed successfully. Defaults to `Pass`. +- `VerdictOnFail`: Verdict to set if any test in the suite fails. Defaults to `Fail`. + ### Run.SingleSliceCreationTime Sends the Slice Creation Time reported by the Slice Manager to InfluxDb. This task will not perform any deployment by itself, and will only read the values for an slice deployed during the experiment pre-run stage.