Implement parsing of go list -deps -json ./... and maintain the rest (#…

…247)

Co-authored-by: Dan Rollo <danrollo@gmail.com>

Jenkinsfile

@@ -33,7 +33,7 @@ dockerizedBuildPipeline(
     withDockerImage(env.DOCKER_IMAGE_ID, {
       withCredentials([usernamePassword(credentialsId: 'policy.s integration account',
         usernameVariable: 'IQ_USERNAME', passwordVariable: 'IQ_PASSWORD')]) {
-        sh 'go list -json -m all | ./nancy iq --iq-application nancy --iq-stage stage-release --iq-username $IQ_USERNAME --iq-token $IQ_PASSWORD --iq-server-url https://policy.ci.sonatype.dev'
+        sh 'go list -json -deps | ./nancy iq --iq-application nancy --iq-stage stage-release --iq-username $IQ_USERNAME --iq-token $IQ_PASSWORD --iq-server-url https://policy.ci.sonatype.dev'
       }
     })
   },

Makefile

@@ -10,6 +10,7 @@ GOLANGCI_VERSION=v1.24.0
 GOLANGCI_LINT_DOCKER=golangci/golangci-lint:$(GOLANGCI_VERSION)
 LINT_CMD=golangci-lint cache status --color always && golangci-lint run --timeout 5m --color always -v --max-same-issues 10
 NANCY_IGNORE=$(shell cat .nancy-ignore | cut -d\# -f 1)
+IT_EXCLUDED_VULNS=CVE-2020-15114,CVE-2020-15136,CVE-2020-15115,CVE-2021-3121
 
 ifeq ($(findstring localbuild,$(CIRCLE_SHELL_ENV)),localbuild)
     DOCKER_CMD=sudo docker
@@ -52,10 +53,12 @@ test: build
 integration-test: build
 	mkdir -p dist
 	cd packages/testdata && GOPATH=. ../../$(BINARY_NAME) sleuth -p Gopkg.lock && cd -
-	go list -json -m all | ./$(BINARY_NAME) sleuth
-	go list -m all | ./$(BINARY_NAME) sleuth
-	go list -json -m all > dist/deps.out && ./$(BINARY_NAME) sleuth < dist/deps.out
-	go list -m all > dist/deps.out && ./$(BINARY_NAME) sleuth < dist/deps.out
+	go list -json -deps | ./$(BINARY_NAME) sleuth
+	go list -json -m all | ./$(BINARY_NAME) sleuth --exclude-vulnerability $(IT_EXCLUDED_VULNS)
+	go list -m all | ./$(BINARY_NAME) sleuth --exclude-vulnerability $(IT_EXCLUDED_VULNS)
+	go list -json -deps > dist/deps.out && ./$(BINARY_NAME) sleuth < dist/deps.out
+	go list -json -m all > dist/deps.out && ./$(BINARY_NAME) sleuth --exclude-vulnerability $(IT_EXCLUDED_VULNS) < dist/deps.out
+	go list -m all > dist/deps.out && ./$(BINARY_NAME) sleuth --exclude-vulnerability $(IT_EXCLUDED_VULNS) < dist/deps.out
 
 build-linux:
 	GOOS=linux GOARCH=amd64 $(GO_BUILD_FLAGS) $(GOBUILD) -o $(BINARY_NAME) -v
@@ -69,7 +72,7 @@ docker-alpine-integration-test: build-linux
     # 2. passes it to the next step that is using this container that only has nancy in it
     # 3. runs nancy using the contents of the exported file with the deps in it. Also assumes that
     #    in ci its likely you have the codebase (thus .nancy-ignore) in the same location you run nancy sleuth
-	go list -json -m all > dist/deps.out
+	go list -json -deps > dist/deps.out
 	echo "cd /tmp && cat /tmp/dist/deps.out | nancy sleuth" > dist/ci.sh
 	chmod +x dist/ci.sh
 	# run the container....using cat with no params keeps it running
@@ -86,7 +89,7 @@ docker-goreleaser-integration-test: build-linux
 	# NANCY_IGNORE is more tomfoolery b/c circleci cant do volume mounts. Use the non-file ignore version but with the contents of
     # the .nancy-ignore. If you were to do this for real you would likely volume mount to your local and it
     # would just use whatever file you actually had.
-	go list -json -m all | $(DOCKER_CMD) run --rm -i sonatypecommunity/nancy:goreleaser-integration-test sleuth -e $(NANCY_IGNORE)
+	go list -json -deps | $(DOCKER_CMD) run --rm -i sonatypecommunity/nancy:goreleaser-integration-test sleuth -e $(NANCY_IGNORE)
 
 docker-integration-tests: docker-alpine-integration-test docker-goreleaser-integration-test
 

README.md

@@ -43,9 +43,9 @@ Usage:
   nancy [command]
 
 Examples:
-  Typical usage will pipe the output of 'go list -json -m all' to 'nancy':
-  go list -json -m all | nancy sleuth [flags]
-  go list -json -m all | nancy iq [flags]
+  Typical usage will pipe the output of 'go list -json -deps' to 'nancy':
+  go list -json -deps | nancy sleuth [flags]
+  go list -json -deps | nancy iq [flags]
 
   If using dep typical usage is as follows :
   nancy sleuth -p Gopkg.lock [flags]
@@ -81,7 +81,7 @@ Usage:
   nancy sleuth [flags]
 
 Examples:
-  go list -json -m all | nancy sleuth --username your_user --token your_token
+  go list -json -deps | nancy sleuth --username your_user --token your_token
   nancy sleuth -p Gopkg.lock --username your_user --token your_token
 
 Flags:
@@ -108,7 +108,7 @@ Usage:
   nancy iq [flags]
 
 Examples:
-  go list -json -m all | nancy iq --iq-application your_public_application_id --iq-server-url http://your_iq_server_url:port --iq-username your_user --iq-token your_token --iq-stage develop
+  go list -json -deps | nancy iq --iq-application your_public_application_id --iq-server-url http://your_iq_server_url:port --iq-username your_user --iq-token your_token --iq-stage develop
   nancy iq -p Gopkg.lock --iq-application your_public_application_id --iq-server-url http://your_iq_server_url:port --iq-username your_user --iq-token your_token --iq-stage develop
 
 Flags:
@@ -134,9 +134,14 @@ Global Flags:
 
 The preferred way to use Nancy is:
 
-- `go list -json -m all | nancy sleuth`
+- `go list -json -deps | nancy sleuth`
 - `nancy sleuth -p /path/to/Gopkg.lock`
 
+If you would like to scan all dependencies, including those that do not end up in the final binary, you can use
+`go list -json -m all` instead:
+
+- `go list -json -m all | nancy sleuth`
+
 #### CI Usage
 
 Here are some additional tools to simplify using Nancy in your CI environment:
@@ -152,7 +157,7 @@ Here are some additional tools to simplify using Nancy in your CI environment:
 
 `nancy` now comes in a boat! For ease of use, we've dockerized `nancy`. To use our Dockerfile:
 
-`go list -json -m all | docker run --rm -i sonatypecommunity/nancy:latest sleuth`
+`go list -json -deps | docker run --rm -i sonatypecommunity/nancy:latest sleuth`
 
 We publish a few different flavors for convenience:
 
@@ -226,7 +231,7 @@ As of Nancy v1.0.17, you can also specify configuration values using environment
 ```shell
 export OSSI_USERNAME=auser@anemailaddress.com
 export OSSI_TOKEN=A4@k3@p1T0k3n
-go list -json -m all | ./nancy sleuth
+go list -json -deps | ./nancy sleuth
 ...
 ```
 
@@ -236,7 +241,7 @@ By default, `nancy` runs in a "quiet" mode, only displaying a list of vulnerable
 You can run `nancy` in a loud manner, showing all components by running:
 
 - `nancy sleuth --loud -p /path/to/your/Gopkg.lock`
-- `go list -json -m all | nancy sleuth --loud`
+- `go list -json -deps | nancy sleuth --loud`
 
 #### Exclude vulnerabilities
 
@@ -250,7 +255,7 @@ We support exclusion of vulnerability either by CVE-ID (ex: `CVE-2018-20303`) or
 ##### Via CLI flag
 
 - `nancy sleuth --exclude-vulnerability CVE-789,bcb0c38d-0d35-44ee-b7a7-8f77183d1ae2 -p /path/to/your/Gopkg.lock`
-- `go list -json -m all | nancy sleuth --exclude-vulnerability CVE-789,bcb0c38d-0d35-44ee-b7a7-8f77183d1ae2`
+- `go list -json -deps | nancy sleuth --exclude-vulnerability CVE-789,bcb0c38d-0d35-44ee-b7a7-8f77183d1ae2`
 
 ##### Via file
 
@@ -259,7 +264,7 @@ By default if a file named `.nancy-ignore` exists in the same directory that nan
 If you would like to define the path to the file you can use the following
 
 - `nancy sleuth --exclude-vulnerability-file=/path/to/your/exclude-file -p /path/to/your/Gopkg.lock`
-- `go list -json -m all | nancy sleuth --exclude-vulnerability-file=/path/to/your/exclude-file`
+- `go list -json -deps | nancy sleuth --exclude-vulnerability-file=/path/to/your/exclude-file`
 
 The file format requires each vulnerability that you want to exclude to be on a separate line. Comments are allowed in the file as well to help provide context when needed. See an example file below.
 
@@ -457,13 +462,13 @@ Count,Package,Is Vulnerable,Num Vulnerabilities,Vulnerabilities
 
 By default, assuming you have an out of the box Nexus IQ Server running, you can run `nancy` like so:
 
-`go list -json -m all | nancy iq --iq-application public-application-id`
+`go list -json -deps | nancy iq --iq-application public-application-id`
 
 It is STRONGLY suggested that you do not do this, and we will warn you on output if you are.
 
 A more logical use of `nancy` against Nexus IQ Server will look like so:
 
-`go list -json -m all | nancy iq --iq-application public-application-id --iq-username nondefaultuser --iq-token yourtoken --iq-server-url http://adifferentserverurl:port --iq-stage develop`
+`go list -json -deps | nancy iq --iq-application public-application-id --iq-username nondefaultuser --iq-token yourtoken --iq-server-url http://adifferentserverurl:port --iq-stage develop`
 
 Options for stage are as follows:
 
@@ -511,7 +516,7 @@ export OSSI_TOKEN=A4@k3@p1T0k3n
 export IQ_USERNAME=nondefaultuser
 export IQ_TOKEN=yourtoken
 export IQ_SERVER=http://adifferentserverurl:port
-go list -json -m all | ./nancy iq --iq-application public-application-id
+go list -json -deps | ./nancy iq --iq-application public-application-id
 ...
 ```
 
@@ -658,7 +663,7 @@ Be aware that even after you add a `replace` directive, `go mod graph` will stil
 You can verify the new version is actually used via the `go list` command:
 ```shell
 $ go mod tidy
-$ go list -m all | grep github.com/gogo/protobuf
+$ go list -deps | grep github.com/gogo/protobuf
 github.com/gogo/protobuf v1.2.1 => github.com/gogo/protobuf v1.3.2
 ```
 You can see the v1.2.1 is replaced with v1.3.2.

go.mod

@@ -37,13 +37,4 @@ require (
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
 )
 
-// fix vulnerability: CVE-2020-15114 in etcd v3.3.13+incompatible
-replace github.com/coreos/etcd => github.com/coreos/etcd v3.3.24+incompatible
-
-// fix vulnerability: CVE-2021-3121 in github.com/gogo/protobuf v1.2.1
-replace github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
-
-// fix vulnerability: SONATYPE-2019-0890 in github.com/pkg/sftp v1.10.1
-replace github.com/pkg/sftp => github.com/pkg/sftp v1.13.0
-
 go 1.13