From 4191f6b51fd62cd3c284e033198a8c3e17393f51 Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Mon, 16 Aug 2021 16:54:48 +0530 Subject: [PATCH] modify exit codes related to scan errors --- pkg/cli/run.go | 20 ++++- pkg/cli/run_test.go | 88 ++++++++++++++++++++++ test/e2e/scan/scan_docker_file_test.go | 5 +- test/e2e/scan/scan_k8s_files_test.go | 5 +- test/e2e/scan/scan_remote_test.go | 18 +++-- test/e2e/scan/scan_rules_filtering_test.go | 12 +-- test/e2e/scan/scan_test.go | 22 +++--- test/e2e/scan/scan_tf_files_test.go | 4 +- test/helper/helper.go | 14 ++-- 9 files changed, 153 insertions(+), 35 deletions(-) diff --git a/pkg/cli/run.go b/pkg/cli/run.go index 676c9f1f2..e26bbd633 100644 --- a/pkg/cli/run.go +++ b/pkg/cli/run.go @@ -207,9 +207,12 @@ func (s *ScanOptions) Run() error { return err } - if !s.configOnly && results.Violations.ViolationStore.Summary.ViolatedPolicies != 0 && flag.Lookup("test.v") == nil { + if !s.configOnly && flag.Lookup("test.v") == nil { os.RemoveAll(tempDir) - os.Exit(3) + exitCode := getExitCode(results) + if exitCode != 0 { + os.Exit(exitCode) + } } return nil } @@ -247,3 +250,16 @@ func (s ScanOptions) writeResults(results runtime.Output) error { return writer.Write(s.outputType, results.Violations, outputWriter) } + +// getExitCode returns appropriate exit code for terrascan based on scan output +func getExitCode(o runtime.Output) int { + if len(o.Violations.ViolationStore.DirScanErrors) > 0 { + if o.Violations.ViolationStore.Summary.ViolatedPolicies > 0 { + return 5 + } + return 4 + } else if o.Violations.ViolationStore.Summary.ViolatedPolicies > 0 { + return 3 + } + return 0 +} diff --git a/pkg/cli/run_test.go b/pkg/cli/run_test.go index 33fb05021..a96e3de0c 100644 --- a/pkg/cli/run_test.go +++ b/pkg/cli/run_test.go @@ -567,3 +567,91 @@ func TestScanOptionsScan(t *testing.T) { }) } } + +func Test_getExitCode(t *testing.T) { + testDirScanErrors := []results.DirScanErr{ + { + IacType: "all", + Directory: "test", + ErrMessage: "error occurred", + }, + } + + testScanSummary := results.ScanSummary{ + ViolatedPolicies: 1, + } + + scanOutputWithDirErrorsOnly := runtime.Output{ + Violations: policy.EngineOutput{ + ViolationStore: &results.ViolationStore{ + DirScanErrors: testDirScanErrors, + }, + }, + } + + scanOutputWithDirErrorsAndViolatedPolicies := runtime.Output{ + Violations: policy.EngineOutput{ + ViolationStore: &results.ViolationStore{ + DirScanErrors: testDirScanErrors, + Summary: testScanSummary, + }, + }, + } + + scanOutputWithViolatedPoliciesOnly := runtime.Output{ + Violations: policy.EngineOutput{ + ViolationStore: &results.ViolationStore{ + Summary: testScanSummary, + }, + }, + } + + type args struct { + o runtime.Output + } + tests := []struct { + name string + args args + want int + }{ + { + name: "has directory scan errors without violated policies", + args: args{ + o: scanOutputWithDirErrorsOnly, + }, + want: 4, + }, + { + name: "has directory scan errors with violated policies", + args: args{ + o: scanOutputWithDirErrorsAndViolatedPolicies, + }, + want: 5, + }, + { + name: "has violated policies but no directory scan errors", + args: args{ + o: scanOutputWithViolatedPoliciesOnly, + }, + want: 3, + }, + { + name: "neither violations nor directory scan errors", + args: args{ + o: runtime.Output{ + Violations: policy.EngineOutput{ + ViolationStore: &results.ViolationStore{}, + }, + }, + }, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getExitCode(tt.args.o); got != tt.want { + t.Errorf("getExitCode() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/test/e2e/scan/scan_docker_file_test.go b/test/e2e/scan/scan_docker_file_test.go index 1be3f39ae..bfb7b1285 100644 --- a/test/e2e/scan/scan_docker_file_test.go +++ b/test/e2e/scan/scan_docker_file_test.go @@ -53,8 +53,9 @@ var _ = Describe("Scan is run for dockerfile directories and files", func() { It("should scan all iac and display violations", func() { scanArgs := []string{scanUtils.ScanCommand, "-d", iacDir} session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...) - // exit code is 3 because iac files in directory has violations - helper.ValidateExitCode(session, scanUtils.ScanTimeout, helper.ExitCodeThree) + // exit code is 5 because iac files in directory has violations + // and directory scan errors + helper.ValidateExitCode(session, scanUtils.ScanTimeout, helper.ExitCodeFive) }) }) }) diff --git a/test/e2e/scan/scan_k8s_files_test.go b/test/e2e/scan/scan_k8s_files_test.go index 488316477..bebab49dd 100644 --- a/test/e2e/scan/scan_k8s_files_test.go +++ b/test/e2e/scan/scan_k8s_files_test.go @@ -53,8 +53,9 @@ var _ = Describe("Scan is run for k8s directories and files", func() { It("should scan will all iac and display violations", func() { scanArgs := []string{scanUtils.ScanCommand, "-d", iacDir} session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...) - // exit code is 3 because iac files in directory has violations - helper.ValidateExitCode(session, scanUtils.ScanTimeout, helper.ExitCodeThree) + // exit code is 5 because iac files in directory has violations + // and directory scan errors + helper.ValidateExitCode(session, scanUtils.ScanTimeout, helper.ExitCodeFive) }) }) }) diff --git a/test/e2e/scan/scan_remote_test.go b/test/e2e/scan/scan_remote_test.go index e12fd30b7..30af67ebb 100644 --- a/test/e2e/scan/scan_remote_test.go +++ b/test/e2e/scan/scan_remote_test.go @@ -139,14 +139,18 @@ var _ = Describe("Scan Command using remote types", func() { It("should download the resource and generate scan results", func() { scanArgs := []string{scanUtils.ScanCommand, "-r", "git", "--remote-url", remoteURL} session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...) - Eventually(session, scanUtils.RemoteScanTimeout).Should(gexec.Exit(helper.ExitCodeThree)) + // exit code is 5 because iac files in directory has violations + // and directory scan errors + Eventually(session, scanUtils.RemoteScanTimeout).Should(gexec.Exit(helper.ExitCodeFive)) }) It("should download the resource and generate scan results", func() { remoteURL := "https://github.com/accurics/KaiMonkey.git//terraform/aws" scanArgs := []string{scanUtils.ScanCommand, "-r", "git", "--remote-url", remoteURL} session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...) - Eventually(session, scanUtils.RemoteScanTimeout).Should(gexec.Exit(helper.ExitCodeThree)) + // exit code is 5 because iac files in directory has violations + // and directory scan errors + Eventually(session, scanUtils.RemoteScanTimeout).Should(gexec.Exit(helper.ExitCodeFive)) }) }) @@ -175,7 +179,8 @@ var _ = Describe("Scan Command using remote types", func() { scanArgs := []string{scanUtils.ScanCommand, "-r", "terraform-registry", "--remote-url", remoteURL} session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...) // has a OR condition because we don't know if there would be violations or not - Eventually(session, scanUtils.RemoteScanTimeout).Should(Or(gexec.Exit(helper.ExitCodeThree), gexec.Exit(helper.ExitCodeZero))) + // there would be directory scan errors due to all iac type + Eventually(session, scanUtils.RemoteScanTimeout).Should(Or(gexec.Exit(helper.ExitCodeFive), gexec.Exit(helper.ExitCodeFour))) }) }) @@ -185,7 +190,8 @@ var _ = Describe("Scan Command using remote types", func() { scanArgs := []string{scanUtils.ScanCommand, "-r", "terraform-registry", "--remote-url", remoteURL} session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...) // has a OR condition because we don't know if there would be violations or not - Eventually(session, scanUtils.RemoteScanTimeout).Should(Or(gexec.Exit(helper.ExitCodeThree), gexec.Exit(helper.ExitCodeZero))) + // there would be directory scan errors due to all iac type + Eventually(session, scanUtils.RemoteScanTimeout).Should(Or(gexec.Exit(helper.ExitCodeFive), gexec.Exit(helper.ExitCodeFour))) }) }) @@ -193,7 +199,7 @@ var _ = Describe("Scan Command using remote types", func() { When("remote type is terraform registry and remote url has a subdirectory", func() { remoteURL := "terraform-aws-modules/security-group/aws//modules/http-80" It("should download the remote registry and generate scan results", func() { - scanArgs := []string{scanUtils.ScanCommand, "-r", "terraform-registry", "--remote-url", remoteURL} + scanArgs := []string{scanUtils.ScanCommand, "-r", "terraform-registry", "--remote-url", remoteURL, "-i", "terraform", "--non-recursive"} session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...) // has a OR condition because we don't know if there would be violations or not Eventually(session, scanUtils.RemoteScanTimeout).Should(Or(gexec.Exit(helper.ExitCodeThree), gexec.Exit(helper.ExitCodeZero))) @@ -203,7 +209,7 @@ var _ = Describe("Scan Command using remote types", func() { When("remote type is git and remote url has a subdirectory", func() { remoteURL := "github.com/terraform-aws-modules/terraform-aws-security-group//modules/http-80" It("should download the remote registry and generate scan results", func() { - scanArgs := []string{scanUtils.ScanCommand, "-r", "git", "--remote-url", remoteURL} + scanArgs := []string{scanUtils.ScanCommand, "-r", "git", "--remote-url", remoteURL, "-i", "terraform", "--non-recursive"} session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...) // has a OR condition because we don't know if there would be violations or not Eventually(session, scanUtils.RemoteScanTimeout).Should(Or(gexec.Exit(helper.ExitCodeThree), gexec.Exit(helper.ExitCodeZero))) diff --git a/test/e2e/scan/scan_rules_filtering_test.go b/test/e2e/scan/scan_rules_filtering_test.go index f59aec6fe..535a670e8 100644 --- a/test/e2e/scan/scan_rules_filtering_test.go +++ b/test/e2e/scan/scan_rules_filtering_test.go @@ -129,18 +129,19 @@ var _ = Describe("Scan command with rule filtering options", func() { Context("severity leve specified is 'low'", func() { Context("iac file has only medium severity violations", func() { It("should report the violations and exit with status code 3", func() { - scanArgs := []string{scanUtils.ScanCommand, "-p", policyDir, "-d", iacDir, "-o", "json", "--severity", "low"} + scanArgs := []string{scanUtils.ScanCommand, "-p", policyDir, "-d", iacDir, "-o", "json", "--severity", "low", "-i", "terraform"} session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...) Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeThree)) }) }) }) - Context("severity leve specified is 'high'", func() { + Context("severity level specified is 'high'", func() { Context("iac files has only medium severity violations", func() { - It("should not report any violation and exit with status code 0", func() { + // there would not no violations but directory scan errors would be present due to all iac scan + It("should not report any violation and exit with status code 4", func() { scanArgs := []string{scanUtils.ScanCommand, "-p", policyDir, "-d", iacDir, "-o", "json", "--severity", "high"} session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...) - Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeZero)) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeFour)) }) }) }) @@ -171,7 +172,8 @@ var _ = Describe("Scan command with rule filtering options", func() { It("should not report any violation and exit with status code 0", func() { scanArgs := []string{scanUtils.ScanCommand, "-p", policyDir, "-d", iacDir, "-o", "json", "--categories", "COMPLIANCE VALIDATION"} session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...) - Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeZero)) + // summary would contain directory scan errors due to all iac scan + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeFour)) }) }) }) diff --git a/test/e2e/scan/scan_test.go b/test/e2e/scan/scan_test.go index e6b72883f..b082170f7 100644 --- a/test/e2e/scan/scan_test.go +++ b/test/e2e/scan/scan_test.go @@ -81,10 +81,10 @@ var _ = Describe("Scan", func() { Describe("scan command is run", func() { Context("when no iac type is provided, terrascan scans with all iac providers", func() { Context("no tf files are present in the working directory", func() { - It("scan the directory and display results", func() { + It("scans the directory with all iac and display results", func() { scanArgs := []string{scanUtils.ScanCommand} session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...) - helper.ValidateExitCode(session, scanUtils.ScanTimeout, helper.ExitCodeZero) + helper.ValidateExitCode(session, scanUtils.ScanTimeout, helper.ExitCodeFour) }) Context("iac loading errors would be displayed in the output, output type is json", func() { It("scan the directory and display results", func() { @@ -106,17 +106,17 @@ var _ = Describe("Scan", func() { }) }) Context("tf files are present in the working directory", func() { - It("should scan the directory, return results and exit with status code 3", func() { + It("should scan the directory, return results and exit with status code 3 as there would no directory scan errors", func() { workDir, err := filepath.Abs(filepath.Join(awsIacRelPath, "aws_ami_violation")) Expect(err).NotTo(HaveOccurred()) - scanArgs := []string{scanUtils.ScanCommand} + scanArgs := []string{scanUtils.ScanCommand, "-i", "terraform", "--non-recursive"} session = helper.RunCommandDir(terrascanBinaryPath, workDir, outWriter, errWriter, scanArgs...) Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeThree)) }) When("tf file present in the dir has no violations", func() { - Context("when there are no violations, terrascan exits with status code 0", func() { + Context("when there are no violations, but has dir scan erros, terrascan exits with status code 4", func() { It("should scan the directory and exit with status code 0", func() { workDir, err := filepath.Abs(filepath.Join(awsIacRelPath, "aws_db_instance_violation")) Expect(err).NotTo(HaveOccurred()) @@ -127,7 +127,7 @@ var _ = Describe("Scan", func() { scanArgs := []string{scanUtils.ScanCommand, "-p", policyDir} session = helper.RunCommandDir(terrascanBinaryPath, workDir, outWriter, errWriter, scanArgs...) - Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeZero)) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeFour)) }) }) }) @@ -194,10 +194,10 @@ var _ = Describe("Scan", func() { When("--iac-version flag is supplied invalid version", func() { Context("default iac type is all and --iac-version would be ignored", func() { - It("should error out and exit with status code 1", func() { + It("should error out and exit with status code 4", func() { scanArgs := []string{scanUtils.ScanCommand, "--iac-version", "test"} session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...) - helper.ValidateExitCode(session, scanUtils.ScanTimeout, helper.ExitCodeZero) + helper.ValidateExitCode(session, scanUtils.ScanTimeout, helper.ExitCodeFour) }) }) }) @@ -276,9 +276,9 @@ var _ = Describe("Scan", func() { It("should scan with the policies and exit with status code 0", func() { scanArgs := []string{scanUtils.ScanCommand, "-p", validPolicyPath1, "-p", validPolicyPath2} session = helper.RunCommandDir(terrascanBinaryPath, workDirPath, outWriter, errWriter, scanArgs...) - // exits with status code 0, because all iac scan should display results - // and the directory doesn't have iac files for violations - Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeZero)) + // exits with status code 4, because there are no iac files for violations but + // would contain directory scan errors + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeFour)) }) }) diff --git a/test/e2e/scan/scan_tf_files_test.go b/test/e2e/scan/scan_tf_files_test.go index 5c857807e..531a50f7c 100644 --- a/test/e2e/scan/scan_tf_files_test.go +++ b/test/e2e/scan/scan_tf_files_test.go @@ -128,7 +128,7 @@ var _ = Describe("Scan is run for terraform files", func() { Context("when iac type is not specified and a directory is specified, it will be scanned will all iac providers", func() { It("should display violations in json format, and should have iac type as 'all'", func() { scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "json"} - scanUtils.RunScanAndAssertGoldenOutputRegex(terrascanBinaryPath, filepath.Join(tfAwsAmiGoldenRelPath, "aws_ami_violation_json_all.txt"), helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanAndAssertGoldenOutputRegex(terrascanBinaryPath, filepath.Join(tfAwsAmiGoldenRelPath, "aws_ami_violation_json_all.txt"), helper.ExitCodeFive, false, true, outWriter, errWriter, scanArgs...) }) }) }) @@ -217,7 +217,7 @@ var _ = Describe("Scan is run for terraform files", func() { It("should display violations in json format", func() { iacDir := filepath.Join(iacRootRelPath, "terraform_recursive") scanArgs := []string{"-i", "terraform", "-p", policyDir, "-d", iacDir, "-o", "json"} - scanUtils.RunScanAndAssertGoldenOutputRegex(terrascanBinaryPath, filepath.Join(tfAwsAmiGoldenRelPath, "aws_ami_violation_json_recursive.txt"), helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanAndAssertGoldenOutputRegex(terrascanBinaryPath, filepath.Join(tfAwsAmiGoldenRelPath, "aws_ami_violation_json_recursive.txt"), helper.ExitCodeFive, false, true, outWriter, errWriter, scanArgs...) }) }) }) diff --git a/test/helper/helper.go b/test/helper/helper.go index f0c7330bb..9f5fdcab4 100644 --- a/test/helper/helper.go +++ b/test/helper/helper.go @@ -43,11 +43,15 @@ import ( const ( // ExitCodeZero represents command exit code 0 - ExitCodeZero = 0 - // ExitCodeOne represents command exit code 0 - ExitCodeOne = 1 - // ExitCodeThree represents command exit code 0 - ExitCodeThree = 3 + ExitCodeZero = iota + // ExitCodeOne represents command exit code 1 + ExitCodeOne + // ExitCodeThree represents command exit code 3 + ExitCodeThree = iota + 1 + // ExitCodeFour represents command exit code 4 + ExitCodeFour + // ExitCodeFive represents command exit code 5 + ExitCodeFive ) var (