⭐️ support upstream communication to fetch policies and store results (…

…#105)

This change adds upstream communication support to cnspec.

- depends on mondoohq/cnquery#286
- depends on mondoohq/cnquery#287
- depends on mondoohq/cnquery#288

.vscode/launch.json

@@ -36,7 +36,6 @@
         "k8s",
         "--discover",
         "all",
-        "--all-namespaces",
         "-f",
         "../cnspec-policies/core/mondoo-kubernetes-security.mql.yaml"
       ]

apps/cnspec/cmd/scan.go

@@ -489,14 +489,30 @@ func (c *scanConfig) loadPolicies() error {
 		return nil
 	}
 
-	return errors.New("Cannot yet resolve policies other than incognito")
+	return nil
 }
 
 func RunScan(config *scanConfig) (*policy.ReportCollection, error) {
-	scanner := scan.NewLocalScanner()
+	opts := []scan.ScannerOption{}
+	if config.UpstreamConfig != nil {
+		opts = append(opts, scan.WithUpstream(config.UpstreamConfig.ApiEndpoint, config.UpstreamConfig.SpaceMrn, config.UpstreamConfig.Plugins))
+	}
+
+	scanner := scan.NewLocalScanner(opts...)
 	ctx := cnquery.SetFeatures(context.Background(), config.Features)
 
-	return scanner.RunIncognito(
+	if config.IsIncognito {
+		return scanner.RunIncognito(
+			ctx,
+			&scan.Job{
+				DoRecord:      config.DoRecord,
+				Inventory:     config.Inventory,
+				Bundle:        config.Bundle,
+				PolicyFilters: config.PolicyNames,
+			})
+	}
+
+	return scanner.Run(
 		ctx,
 		&scan.Job{
 			DoRecord:      config.DoRecord,

cli/reporter/print_compact.go

@@ -2,7 +2,7 @@ package reporter
 
 import (
 	"fmt"
-	io "io"
+	"io"
 	"sort"
 	"strings"
 	"unicode/utf8"
@@ -34,6 +34,11 @@ type defaultReporter struct {
 }
 
 func (r *defaultReporter) print() error {
+	// catch case where the scan was not successful and no bundle was fetched from server
+	if r.data == nil || r.data.Bundle == nil {
+		return nil
+	}
+
 	r.bundle = r.data.Bundle.ToMap()
 
 	// sort assets by name, to make it more intuitive
@@ -219,8 +224,14 @@ func (r *defaultReporter) printAssetSummary(assetMrn string, asset *policy.Asset
 		r.out.Write([]byte{'\n'})
 	}
 
-	if !r.IsIncognito {
-		panic("PROVIDE UPSTREAM URL")
+	if !r.IsIncognito && report.Url != "" || asset.Url != "" {
+		r.out.Write([]byte("Report URL: "))
+		url := report.Url
+		if url == "" {
+			url = asset.Url
+		}
+		r.out.Write([]byte(url))
+		r.out.Write([]byte{'\n'})
 	}
 }
 

go.mod

@@ -26,9 +26,10 @@ require (
 	github.com/spf13/pflag v1.0.6-0.20201009195203-85dd5c8bc61c
 	github.com/spf13/viper v1.13.0
 	github.com/stretchr/testify v1.8.0
-	go.mondoo.com/cnquery v0.0.0-20221014220524-5bb7420b2901
+	go.mondoo.com/cnquery v0.0.0-20221016203430-4b6d2815157f
 	go.mondoo.com/ranger-rpc v0.5.1-0.20220923135836-9e7732899d34
 	go.opentelemetry.io/otel v1.11.0
+	golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7
 	google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/yaml.v3 v3.0.1
@@ -405,7 +406,6 @@ require (
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b // indirect
 	golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect
-	golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 // indirect
 	golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect
 	golang.org/x/term v0.0.0-20220919170432-7a66f970e087 // indirect
 	golang.org/x/text v0.3.7 // indirect

go.sum

@@ -1566,8 +1566,8 @@ go.etcd.io/etcd v0.0.0-20200513171258-e048e166ab9c/go.mod h1:xCI7ZzBfRuGgBXyXO6y
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
-go.mondoo.com/cnquery v0.0.0-20221014220524-5bb7420b2901 h1:BR3orlzkv6DaoUwOH4z0eDB/PoEwh7UAICSpkpyp86A=
-go.mondoo.com/cnquery v0.0.0-20221014220524-5bb7420b2901/go.mod h1:E7+94D2X1SMG2Gc7HL5P6FONssPF9i0cXlK9Vr5e+G4=
+go.mondoo.com/cnquery v0.0.0-20221016203430-4b6d2815157f h1:1ZIFCJcWuc0w2h/d5n4Gym04PjjD3cp2f9QNBTUi76Q=
+go.mondoo.com/cnquery v0.0.0-20221016203430-4b6d2815157f/go.mod h1:E7+94D2X1SMG2Gc7HL5P6FONssPF9i0cXlK9Vr5e+G4=
 go.mondoo.com/ranger-rpc v0.5.1-0.20220923135836-9e7732899d34 h1:mtPZ1J+nRI/ivV+n41bjIwY6Rfxb2Jf49svZSQMGHIA=
 go.mondoo.com/ranger-rpc v0.5.1-0.20220923135836-9e7732899d34/go.mod h1:3YKcqFrlPgaB4FZ4EoLgdmRtwMQdO7RoAkZYFn+F1eY=
 go.mozilla.org/mozlog v0.0.0-20170222151521-4bb13139d403/go.mod h1:jHoPAGnDrCy6kaI2tAze5Prf0Nr0w/oNkROt2lw3n3o=

internal/datalakes/inmemory/inmemory.go

@@ -49,7 +49,7 @@ func WithDb(resolvedPolicyCache *ResolvedPolicyCache, f func(*Db, *policy.LocalS
 }
 
 // Prefixes for all keys that are stored in the cache.
-// Prevent collissions by creating namespaces for different types of data.
+// Prevent collisions by creating namespaces for different types of data.
 const (
 	dbIDQuery          = "q\x00"
 	dbIDPolicy         = "p\x00"