Bug fixes for test results

tests_e2e/orchestrator/scripts/run-scenarios

@@ -45,5 +45,5 @@ lisa \
   --log_path "$lisa_logs" \
   --working_path "$lisa_logs" \
   -v subscription_id:"$SUBSCRIPTION_ID" \
-  -v identity_file:"$HOME/.ssh/id_rsa" \
-  || true # force a success exit code to allow execution to continue when a test fails
+  -v identity_file:"$HOME/.ssh/id_rsa"
+

tests_e2e/pipeline/scripts/execute_tests.sh

@@ -14,6 +14,10 @@ docker pull waagenttests.azurecr.io/waagenttests:latest
 sudo chown 1000 "$BUILD_SOURCESDIRECTORY"
 sudo chown 1000 "$BUILD_ARTIFACTSTAGINGDIRECTORY"
 
+# A test failure will cause automation to exit with an error code and we don't want this script to stop so we force the command
+# to succeed and capture the exit code to return it at the end of the script.
+echo "exit 0" > /tmp/exit.sh
+
 docker run --rm \
       --volume "$BUILD_SOURCESDIRECTORY:/home/waagent/WALinuxAgent" \
       --volume "$DOWNLOADSSHKEY_SECUREFILEPATH:/home/waagent/id_rsa" \
@@ -23,7 +27,8 @@ docker run --rm \
       --env AZURE_CLIENT_SECRET \
       --env AZURE_TENANT_ID \
       waagenttests.azurecr.io/waagenttests \
-      bash --login -c '$HOME/WALinuxAgent/tests_e2e/orchestrator/scripts/run-scenarios'
+      bash --login -c '$HOME/WALinuxAgent/tests_e2e/orchestrator/scripts/run-scenarios' \
+|| echo "exit $?" > /tmp/exit.sh
 
 # Retake ownership of the source and staging directory (note that the former does not need to be done recursively)
 sudo chown "$USER" "$BUILD_SOURCESDIRECTORY"
@@ -43,3 +48,6 @@ sudo find "$BUILD_ARTIFACTSTAGINGDIRECTORY" -exec chown "$USER" {} \;
 #
 mv "$BUILD_ARTIFACTSTAGINGDIRECTORY"/lisa/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]/*/* "$BUILD_ARTIFACTSTAGINGDIRECTORY"/lisa
 rm -r "$BUILD_ARTIFACTSTAGINGDIRECTORY"/lisa/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]
+
+cat /tmp/exit.sh
+bash /tmp/exit.sh

tests_e2e/pipeline/templates/execute-tests.yml

@@ -26,20 +26,23 @@ jobs:
 
       - bash: $(Build.SourcesDirectory)/tests_e2e/pipeline/scripts/execute_tests.sh
         displayName: "Execute tests"
+        continueOnError: true
         env:
           # Add all KeyVault secrets explicitly as they're not added by default to the environment vars
           AZURE_CLIENT_ID: $(AZURE-CLIENT-ID)
           AZURE_CLIENT_SECRET: $(AZURE-CLIENT-SECRET)
           AZURE_TENANT_ID: $(AZURE-TENANT-ID)
           SUBSCRIPTION_ID: $(SUBSCRIPTION-ID)
 
+      - publish: $(Build.ArtifactStagingDirectory)
+        artifact: 'artifacts'
+        displayName: 'Publish test artifacts'
+
       - task: PublishTestResults@2
+        displayName: 'Publish test results'
         inputs:
           testResultsFormat: 'JUnit'
-          testResultsFiles: '**/*junit.xml'
+          testResultsFiles: 'lisa/agent.junit.xml'
           searchFolder: $(Build.ArtifactStagingDirectory)
-          testRunTitle: 'Publish test results'
+          failTaskOnFailedTests: true
 
-      - publish: $(Build.ArtifactStagingDirectory)
-        artifact: 'test-logs'
-        displayName: 'Publish test logs'

tests_e2e/scenarios/runbooks/daily.yml

@@ -72,7 +72,7 @@ concurrency: 10
 
 notifier:
   - type: env_stats
-  - type: junit
+  - type: agent.junit
 
 include:
   - path: ./include/ssh_proxy.yml

tests_e2e/scenarios/runbooks/samples/existing_vm.yml

@@ -59,7 +59,7 @@ platform:
 
 notifier:
   - type: env_stats
-  - type: junit
+  - type: agent.junit
 
 include:
   - path: ../include/ssh_proxy.yml

tests_e2e/scenarios/testsuites/agent_junit.py

@@ -0,0 +1,50 @@
+# Microsoft Azure Linux Agent
+#
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from dataclasses import dataclass
+from typing import Type
+
+from dataclasses_json import dataclass_json
+
+from lisa.notifiers.junit import JUnit
+from lisa import schema
+from lisa.messages import (
+    MessageBase,
+    TestResultMessage,
+)
+
+
+@dataclass_json()
+@dataclass
+class AgentJUnitSchema(schema.Notifier):
+    path: str = "agent.junit.xml"
+
+
+class AgentJUnit(JUnit):
+    @classmethod
+    def type_name(cls) -> str:
+        return "agent.junit"
+
+    @classmethod
+    def type_schema(cls) -> Type[schema.TypedSchema]:
+        return AgentJUnitSchema
+
+    def _received_message(self, message: MessageBase) -> None:
+        if isinstance(message, TestResultMessage):
+            distro = message.information.get('distro_version')
+            if distro is not None:
+                message.name = distro
+        super()._received_message(message)