Merge pull request #527 from patilpankaj212/support-junitXML

Adds support for junit xml output

pkg/cli/register.go

@@ -36,7 +36,7 @@ func RegisterCommand(baseCommand *cobra.Command, command *cobra.Command) {
 func Execute() {
 	rootCmd.PersistentFlags().StringVarP(&LogLevel, "log-level", "l", "info", "log level (debug, info, warn, error, panic, fatal)")
 	rootCmd.PersistentFlags().StringVarP(&LogType, "log-type", "x", "console", "log output type (console, json)")
-	rootCmd.PersistentFlags().StringVarP(&OutputType, "output", "o", "human", "output type (human, json, yaml, xml)")
+	rootCmd.PersistentFlags().StringVarP(&OutputType, "output", "o", "human", "output type (human, json, yaml, xml, junit-xml)")
 	rootCmd.PersistentFlags().StringVarP(&ConfigFile, "config-path", "c", "", "config file path")
 
 	// Function to execute before processing commands

pkg/policy/opa/engine.go

@@ -416,6 +416,9 @@ func (e *Engine) Evaluate(engineInput policy.EngineInput) (policy.EngineOutput,
 
 	// add the rule count of the policy engine to result summary
 	e.results.ViolationStore.Summary.TotalPolicies += e.stats.ruleCount
+
+	// add the time taken to the result summary
+	e.results.ViolationStore.Summary.TotalTime += int64(e.stats.runTime)
 	return e.results, nil
 }
 

pkg/results/types.go

@@ -55,6 +55,7 @@ type ScanSummary struct {
 	LowCount             int    `json:"low" yaml:"low" xml:"low,attr"`
 	MediumCount          int    `json:"medium" yaml:"medium" xml:"medium,attr"`
 	HighCount            int    `json:"high" yaml:"high" xml:"high,attr"`
+	TotalTime            int64  `json:"-" yaml:"-" xml:"-"`
 }
 
 // Add adds two ViolationStores

pkg/writer/junit_xml.go

@@ -0,0 +1,191 @@
+/*
+    Copyright (C) 2020 Accurics, Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+package writer
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+
+	"github.com/accurics/terrascan/pkg/policy"
+	"github.com/accurics/terrascan/pkg/results"
+	"github.com/accurics/terrascan/pkg/version"
+)
+
+const (
+	junitXMLFormat supportedFormat = "junit-xml"
+	testSuiteName  string          = "TERRASCAN_POLICY_SUITE"
+	testSuitesName string          = "TERRASCAN_POLICY_SUITES"
+	testNameFormat string          = `[ERROR] resource: "%s" at line: %d, violates: RULE - %s`
+)
+
+// JUnitTestSuites is a collection of JUnit test suites.
+type JUnitTestSuites struct {
+	XMLName  xml.Name `xml:"testsuites"`
+	Tests    int      `xml:"tests,attr"`
+	Name     string   `xml:"name,attr"`
+	Failures int      `xml:"failures,attr"`
+	Time     string   `xml:"time,attr"`
+	Suites   []JUnitTestSuite
+}
+
+// JUnitTestSuite is a single JUnit test suite which may contain many testcases.
+type JUnitTestSuite struct {
+	XMLName    xml.Name        `xml:"testsuite"`
+	Tests      int             `xml:"tests,attr"`
+	Failures   int             `xml:"failures,attr"`
+	Time       string          `xml:"time,attr"`
+	Name       string          `xml:"name,attr"`
+	Package    string          `xml:"package,attr"`
+	Properties []JUnitProperty `xml:"properties>property,omitempty"`
+	TestCases  []JUnitTestCase
+}
+
+// JUnitTestCase is a single test case with its result.
+type JUnitTestCase struct {
+	XMLName   xml.Name `xml:"testcase"`
+	Classname string   `xml:"classname,attr"`
+	Name      string   `xml:"name,attr"`
+	Severity  string   `xml:"severity,attr"`
+	Category  string   `xml:"category,attr"`
+	// omit empty time because today we do not have this data
+	Time        string            `xml:"time,attr,omitempty"`
+	SkipMessage *JUnitSkipMessage `xml:"skipped,omitempty"`
+	Failure     *JUnitFailure     `xml:"failure,omitempty"`
+}
+
+// JUnitSkipMessage contains the reason why a testcase was skipped.
+type JUnitSkipMessage struct {
+	Message string `xml:"message,attr"`
+}
+
+// JUnitProperty represents a key/value pair used to define properties.
+type JUnitProperty struct {
+	Name  string `xml:"name,attr"`
+	Value string `xml:"value,attr"`
+}
+
+// JUnitFailure contains data related to a failed test.
+type JUnitFailure struct {
+	Message  string `xml:"message,attr"`
+	Type     string `xml:"type,attr"`
+	Contents string `xml:",chardata"`
+}
+
+func newJunitTestSuites(summary results.ScanSummary) JUnitTestSuites {
+	return JUnitTestSuites{
+		Tests:    summary.TotalPolicies,
+		Name:     testSuitesName,
+		Failures: summary.ViolatedPolicies,
+		Time:     fmt.Sprint(summary.TotalTime),
+	}
+}
+
+func newJunitTestSuite(summary results.ScanSummary) JUnitTestSuite {
+	return JUnitTestSuite{
+		Name:     testSuiteName,
+		Tests:    summary.TotalPolicies,
+		Time:     fmt.Sprint(summary.TotalTime),
+		Failures: summary.ViolatedPolicies,
+		Package:  summary.ResourcePath,
+		Properties: []JUnitProperty{
+			{
+				Name:  "Terrascan Version",
+				Value: version.Get(),
+			},
+		}}
+}
+
+func init() {
+	RegisterWriter(junitXMLFormat, JUnitXMLWriter)
+}
+
+// JUnitXMLWriter writes scan summary in junit xml format
+func JUnitXMLWriter(data interface{}, writer io.Writer) error {
+	output, ok := data.(policy.EngineOutput)
+	if !ok {
+		return fmt.Errorf("incorrect input for JunitXML writer, supported type is policy.EngineOutput")
+	}
+
+	junitXMLOutput := convert(output)
+
+	return XMLWriter(junitXMLOutput, writer)
+}
+
+// convert is helper func to convert engine output to JUnitTestSuites
+func convert(output policy.EngineOutput) JUnitTestSuites {
+	testSuites := newJunitTestSuites(output.Summary)
+	// since we have a single suite for now, a suite will have same data as in root level element testsuites
+	suite := newJunitTestSuite(output.Summary)
+
+	tests := violationsToTestCases(output.ViolationStore.Violations, false)
+	if tests != nil {
+		suite.TestCases = append(suite.TestCases, tests...)
+	}
+
+	skippedTests := violationsToTestCases(output.ViolationStore.SkippedViolations, true)
+	if skippedTests != nil {
+		suite.TestCases = append(suite.TestCases, skippedTests...)
+	}
+
+	testSuites.Suites = append(testSuites.Suites, suite)
+
+	return testSuites
+}
+
+// violationsToTestCases is helper func to convert scan violations to JunitTestCases
+func violationsToTestCases(violations []*results.Violation, isSkipped bool) []JUnitTestCase {
+	testCases := make([]JUnitTestCase, 0)
+	for _, v := range violations {
+		var testCase JUnitTestCase
+		if isSkipped {
+			testCase = JUnitTestCase{Failure: new(JUnitFailure), SkipMessage: new(JUnitSkipMessage)}
+			testCase.SkipMessage.Message = v.Comment
+		} else {
+			testCase = JUnitTestCase{Failure: new(JUnitFailure)}
+		}
+		testCase.Classname = v.File
+		testCase.Name = fmt.Sprintf(testNameFormat, v.ResourceName, v.LineNumber, v.RuleID)
+		testCase.Severity = v.Severity
+		testCase.Category = v.Category
+		// since junitXML doesn't contain the attributes we want to show as violations
+		// we would add details of violations in the failure message
+		testCase.Failure.Message = getViolationString(*v)
+		testCases = append(testCases, testCase)
+	}
+	return testCases
+}
+
+// getViolationString is used to get violation details as string
+func getViolationString(v results.Violation) string {
+	resourceName := v.ResourceName
+	if resourceName == "" {
+		resourceName = `""`
+	}
+
+	out := fmt.Sprintf("%s: %s, %s: %s, %s: %d, %s: %s, %s: %s, %s: %s, %s: %s, %s: %s, %s: %s",
+		"Description", v.Description,
+		"File", v.File,
+		"Line", v.LineNumber,
+		"Severity", v.Severity,
+		"Rule Name", v.RuleName,
+		"Rule ID", v.RuleID,
+		"Resource Name", resourceName,
+		"Resource Type", v.ResourceType,
+		"Category", v.Category)
+	return out
+}

pkg/writer/junit_xml_test.go

@@ -0,0 +1,167 @@
+/*
+    Copyright (C) 2020 Accurics, Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+package writer
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/accurics/terrascan/pkg/policy"
+	"github.com/accurics/terrascan/pkg/results"
+	"github.com/accurics/terrascan/pkg/version"
+)
+
+func TestJUnitXMLWriter(t *testing.T) {
+	testOutput := fmt.Sprintf(`
+<testsuites tests="566" name="TERRASCAN_POLICY_SUITES" failures="1" time="0">
+  <testsuite tests="566" failures="1" time="0" name="TERRASCAN_POLICY_SUITE" package="test">
+    <properties>
+      <property name="Terrascan Version" value="%s"></property>
+    </properties>
+    <testcase classname="modules/m1/main.tf" name="[ERROR] resource: &#34;bucket&#34; at line: 20, violates: RULE - AWS.S3Bucket.DS.High.1043" severity="HIGH" category="S3">
+      <failure message="Description: S3 bucket Access is allowed to all AWS Account Users., File: modules/m1/main.tf, Line: 20, Severity: HIGH, Rule Name: s3EnforceUserACL, Rule ID: AWS.S3Bucket.DS.High.1043, Resource Name: bucket, Resource Type: aws_s3_bucket, Category: S3" type=""></failure>
+    </testcase>
+    <testcase classname="modules/m1/main.tf" name="[ERROR] resource: &#34;bucket&#34; at line: 20, violates: RULE - AWS.S3Bucket.DS.High.1043" severity="HIGH" category="S3">
+      <skipped message=""></skipped>
+      <failure message="Description: S3 bucket Access is allowed to all AWS Account Users., File: modules/m1/main.tf, Line: 20, Severity: HIGH, Rule Name: s3EnforceUserACL, Rule ID: AWS.S3Bucket.DS.High.1043, Resource Name: bucket, Resource Type: aws_s3_bucket, Category: S3" type=""></failure>
+    </testcase>
+  </testsuite>
+</testsuites>
+  `, version.Get())
+
+	testOutputNoViolations := fmt.Sprintf(`
+<testsuites tests="550" name="TERRASCAN_POLICY_SUITES" failures="1" time="0">
+  <testsuite tests="550" failures="1" time="0" name="TERRASCAN_POLICY_SUITE" package="test_resource_path">
+    <properties>
+      <property name="Terrascan Version" value="%s"></property>
+    </properties>
+  </testsuite>
+</testsuites>
+	`, version.Get())
+
+	type args struct {
+		data interface{}
+	}
+	tests := []struct {
+		name       string
+		args       args
+		wantWriter string
+		wantErr    bool
+	}{
+		{
+			name: "incorrect input for JunitXMLWriter",
+			args: args{
+				// some invalid data
+				data: struct{ Name string }{Name: "test"},
+			},
+			wantErr: true,
+		},
+		{
+			name: "data with violations and skipped violations",
+			args: args{
+				data: violationsInput,
+			},
+			wantWriter: testOutput,
+		},
+		{
+			name: "data with no violations",
+			args: args{
+				policy.EngineOutput{
+					ViolationStore: &results.ViolationStore{
+						Summary: results.ScanSummary{
+							ResourcePath:     "test_resource_path",
+							IacType:          "k8s",
+							Timestamp:        "2020-12-12 11:21:29.902796 +0000 UTC",
+							TotalPolicies:    550,
+							LowCount:         0,
+							MediumCount:      0,
+							HighCount:        1,
+							ViolatedPolicies: 1,
+						},
+					},
+				},
+			},
+			wantWriter: testOutputNoViolations,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			writer := &bytes.Buffer{}
+			if err := JUnitXMLWriter(tt.args.data, writer); (err != nil) != tt.wantErr {
+				t.Errorf("JUnitXMLWriter() got error = %v, wantErr = %v", err, tt.wantErr)
+				return
+			}
+			if gotWriter := writer.String(); !strings.EqualFold(strings.TrimSpace(gotWriter), strings.TrimSpace(tt.wantWriter)) {
+				t.Errorf("JUnitXMLWriter() got = %v, want = %v", gotWriter, tt.wantWriter)
+			}
+		})
+	}
+}
+
+func TestGetViolationString(t *testing.T) {
+	type args struct {
+		v results.Violation
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "violation with all fields",
+			args: args{
+				v: results.Violation{
+					RuleName:     "RuleA",
+					Description:  "test rule",
+					RuleID:       "Rule.A",
+					Severity:     "MEDIUM",
+					Category:     "A",
+					ResourceName: "aws_resource",
+					ResourceType: "some_resource_type",
+					File:         "file.yaml",
+					LineNumber:   1,
+				},
+			},
+			want: "Description: test rule, File: file.yaml, Line: 1, Severity: MEDIUM, Rule Name: RuleA, Rule ID: Rule.A, Resource Name: aws_resource, Resource Type: some_resource_type, Category: A",
+		},
+		{
+			name: "violation with all fields, blank resource name",
+			args: args{
+				v: results.Violation{
+					RuleName:     "RuleB",
+					Description:  "test rule 2",
+					RuleID:       "Rule.B",
+					Severity:     "HIGH",
+					Category:     "B",
+					ResourceType: "test_resource_type",
+					File:         "file1.yaml",
+					LineNumber:   2,
+				},
+			},
+			want: `Description: test rule 2, File: file1.yaml, Line: 2, Severity: HIGH, Rule Name: RuleB, Rule ID: Rule.B, Resource Name: "", Resource Type: test_resource_type, Category: B`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getViolationString(tt.args.v); got != tt.want {
+				t.Errorf("getViolationString() got = %v, want = %v", got, tt.want)
+			}
+		})
+	}
+}