Merge pull request #6 from EVOLVED-5G/robot_fw

Robot fw

CHANGELOG.md

@@ -1,3 +1,7 @@
+**13/05/2022** [Version 3.4.0]
+
+ - Add Robot Framework task
+
 **04/05/2022** [Version 3.3.1]
 
  - Fix SliceId expansion, exception on SingleSliceCreationTime

Executor/Tasks/Run/__init__.py

@@ -13,3 +13,4 @@
 from .rest_api import RestApi
 from .upgrade_verdict import UpgradeVerdict
 from .evaluate import Evaluate
+from .robot_framework import RobotFramework

Executor/Tasks/Run/robot_framework.py

@@ -0,0 +1,94 @@
+from Task import Task
+from Helper import Cli, Level
+from os.path import abspath, join, exists
+from datetime import datetime, timezone
+import xml.etree.ElementTree as ET
+
+
+class RobotFramework(Task):
+    def __init__(self, logMethod, parent, params):
+        super().__init__("Robot Framework", parent, params, logMethod, None)
+        self.paramRules = {
+            'Executable': (None, True),
+            'Paths': (None, True),
+            'CWD': (None, True),
+            'GatherResults': (True, False),
+            'Identifier': (None, False),
+            'VerdictOnPass': ("Pass", False),
+            'VerdictOnFail': ("Fail", False)
+        }
+
+    def Run(self):
+        from Executor import Verdict
+
+        gatherResults = self.params['GatherResults']
+        paths = self.params['Paths']
+        if isinstance(paths, str):
+            paths = [paths]
+        identifier = self.params['Identifier']  # Save to different folders, in case we run multiple instances in a row
+        identifier = identifier if identifier is not None else f'RobotFw{datetime.now(timezone.utc).strftime("%H%M%S")}'
+        tempFolder = join(abspath(self.parent.TempFolder), identifier)
+        onPass = self.GetVerdictFromName(self.params["VerdictOnPass"])
+        onFail = self.GetVerdictFromName(self.params["VerdictOnFail"])
+
+        parameters = [self.params['Executable'], '--xunit', 'xunit.xml']
+        if gatherResults:
+            parameters.extend(['--outputdir', tempFolder])
+        parameters.extend(paths)
+
+        try:
+            self.Log(Level.INFO, "Executing Robot Framework tests")
+            cli = Cli(parameters, self.params['CWD'], self.Log)
+            cli.Execute()
+            self.Log(Level.INFO, "Robot Framework tests finished")
+        except Exception as e:
+            self.SetVerdictOnError()
+            raise RuntimeError(f"Exception while executing Robot Framework: {e}") from e
+
+        fullXml = join(tempFolder, 'output.xml')
+        if onPass != Verdict.NotSet or onFail != Verdict.NotSet:
+            try:
+                self.Log(Level.INFO, "Generating verdict from test results")
+                xml = ET.parse(fullXml)
+                total = xml.getroot().find("./statistics/total/stat")
+                if total is not None:
+                    success = int(total.attrib['pass'])
+                    fail = int(total.attrib['fail'])
+                    skip = int(total.attrib['skip'])
+                    self.Verdict = onPass if fail == 0 else onFail
+                    self.Log(Level.INFO,
+                             f"Verdict set to {self.Verdict.name} (pass: {success}, fail: {fail}, skip: {skip})")
+                else:
+                    raise RuntimeError("Could not find total statistics in output.xml")
+            except Exception as e:
+                self.SetVerdictOnError()
+                raise RuntimeError(f"Exception while generating verdict: {e}") from e
+
+        else:
+            self.Log(Level.INFO, "Skipping Verdict generation")
+
+        if gatherResults:
+            try:
+                self.Log(Level.INFO, "Gathering test report files")
+                found = []
+                for kind, path in [('Output', fullXml),
+                                   ('Log', join(tempFolder, 'log.html')),
+                                   ('Report', join(tempFolder, 'report.html')),
+                                   ('xUnit compatible', join(tempFolder, 'xunit.xml'))]:
+                    if exists(path):
+                        found.append(path)
+                    else:
+                        self.Log(Level.WARNING, f'Could not retrieve {kind} file ({path})')
+                if len(found) > 0:
+                    from Helper import Compress
+                    path = join(abspath(self.parent.TempFolder), f"{identifier}.zip")
+                    Compress.Zip(found, path, flat=True)
+                    self.parent.GeneratedFiles.append(path)
+                    self.Log(Level.INFO, f"Report files compressed to '{identifier}.zip'")
+                else:
+                    self.Log(Level.WARNING, "No report files found, skipping zip creation.")
+            except Exception as e:
+                self.SetVerdictOnError()
+                raise RuntimeError(f"Exception while gathering generated files: {e}") from e
+        else:
+            self.Log(Level.INFO, "'GatherResults' disabled: Reports will be generated in the CWD folder")

README.md

@@ -577,6 +577,23 @@ success response (2xx). Set to `None` to disable the check.
 - `Timeout`: Maximum time in seconds to wait for a response
 - `Headers`: Additional headers to add to the request
 
+### Run.RobotFramework
+Execute one or more test suites using an external Robot Framework instance. It is recommended to store and configure
+Robot Framework in a dedicated virtualenv, where all the required dependencies (for example `robotframework-requests`)
+are also installed. Configuration values:
+- `Executable`: Absolute path to the Robot Framework executable. On a pre-configured virtualenv this file is usually
+`<venv>/bin/robot` or `<venv>/Scripts/robot.exe`
+- `Paths`: Either a single path (string) or a list of paths, each with the location of one of the test suites to run.
+- `CWD`: Working directory, usually the root folder where the test suites are stored. If `GatherResults` is set to
+`False` the generated report files will be left in this folder.
+- `GatherResults`: Whether or not to store the generated files along with other files created by the experiment.
+These reports will be compressed in a single zip file identified by the `Identifier` parameter. `True` by default.
+- `Identifier`: Name used to identify a particular Robot Framework execution, in order to avoid overwriting results
+for TestCases that include multiple invocations. If not set, it will be automatically generated from the time as
+`RobotFwHHMMSS`, where HHMMSS corresponds to the hour, minutes and seconds in UTC.
+- `VerdictOnPass`: Verdict to set if all tests are completed successfully. Defaults to `Pass`.
+- `VerdictOnFail`: Verdict to set if any test in the suite fails. Defaults to `Fail`.
+
 ### Run.SingleSliceCreationTime
 Sends the Slice Creation Time reported by the Slice Manager to InfluxDb. This task will not perform any deployment
  by itself, and will only read the values for an slice deployed during the experiment pre-run stage.