Merge branch 'main' into es-2122-name-fixes

deepfence · Feb 20, 2024 · 342bd88 · 342bd88
2 parents a6bd6a4 + 188466a
commit 342bd88
Show file tree

Hide file tree

Showing 49 changed files with 1,124 additions and 712 deletions.
diff --git a/Makefile b/Makefile
@@ -16,7 +16,7 @@ export DEEPFENCE_FARGATE_DIR=$(DEEPFENCE_AGENT_DIR)/fargate
 export IMAGE_REPOSITORY?=deepfenceio
 export DF_IMG_TAG?=latest
 export IS_DEV_BUILD?=false
-export VERSION?="2.1.0"
+export VERSION?="2.1.1"
 
 default: bootstrap console_plugins agent console fargate-local
 

diff --git a/deepfence_agent/fargate/Dockerfile.fargate b/deepfence_agent/fargate/Dockerfile.fargate
@@ -115,7 +115,7 @@ LABEL deepfence.role=system
 
 WORKDIR /
 
-COPY fargate/deepfence-agent-bin-2.1.0 /deepfence
+COPY fargate/deepfence-agent-bin-2.1.1 /deepfence
 
 COPY fargate/deepfence-entry-point-scratch.sh deepfence/usr/local/bin/deepfence-entry-point-scratch.sh 
 COPY --from=builder /tmp/rootfs/bin/curl /deepfence/bin/curl

diff --git a/deepfence_agent/tools/apache/scope/probe/host/reporter.go b/deepfence_agent/tools/apache/scope/probe/host/reporter.go
@@ -10,6 +10,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/deepfence/ThreatMapper/deepfence_utils/log"
 	dfUtils "github.com/deepfence/df-utils"
 	"github.com/deepfence/df-utils/cloud_metadata"
 	"github.com/weaveworks/scope/report"
@@ -107,6 +108,8 @@ func getCloudMetadata(cloudProvider string) (string, cloud_metadata.CloudMetadat
 
 func (r *Reporter) updateCloudMetadata(cloudProvider string) {
 	cloudProvider, cloudMetadata := getCloudMetadata(cloudProvider)
+	log.Info().Msgf("Cloud metadata: %v", cloudMetadata)
+
 	r.cloudMeta.mtx.Lock()
 	r.cloudMeta.cloudProvider = cloudProvider
 	r.cloudMeta.cloudMetadata = cloudMetadata

diff --git a/deepfence_agent/tools/apache/scope/probe/kubernetes/kubernetes_cluster.go b/deepfence_agent/tools/apache/scope/probe/kubernetes/kubernetes_cluster.go
@@ -1,44 +1,87 @@
 package kubernetes
 
 import (
+	"sync"
 	"time"
 
+	"github.com/deepfence/ThreatMapper/deepfence_utils/log"
 	"github.com/deepfence/df-utils/cloud_metadata"
 	"github.com/weaveworks/scope/report"
 )
 
 // KubernetesClusterResource represents a Kubernetes cluster
 type KubernetesClusterResource interface {
 	GetNode() report.TopologyNode
+	GetTopology() report.Topology
+	Stop()
 }
 
 type kubernetesCluster struct {
-	cloudProvider  string
-	cloudAccountID string
+	k8sTopology report.Topology
+	stopRefresh chan bool
+	sync.RWMutex
 }
 
 // NewKubernetesClusterResource creates a new Cluster node
 func NewKubernetesClusterResource() KubernetesClusterResource {
-	metadata := cloud_metadata.GetCloudMetadata()
-	return &kubernetesCluster{cloudProvider: metadata.CloudProvider, cloudAccountID: metadata.AccountID}
+	k8sCluster := kubernetesCluster{stopRefresh: make(chan bool)}
+	k8sCluster.cacheK8sTopology()
+	go k8sCluster.refresh()
+	return &k8sCluster
+}
+
+func (k *kubernetesCluster) Stop() {
+	k.stopRefresh <- true
+}
+
+func (k *kubernetesCluster) refresh() {
+	ticker := time.NewTicker(6 * time.Hour)
+	for {
+		select {
+		case <-ticker.C:
+			k.cacheK8sTopology()
+		case <-k.stopRefresh:
+			return
+		}
+	}
+}
+
+func (k *kubernetesCluster) cacheK8sTopology() {
+	k.Lock()
+	defer k.Unlock()
+
+	k.k8sTopology = report.MakeTopology()
+	node := k.GetNode()
+	//cloudProviderNodeId = node.Parents.CloudProvider
+	k.k8sTopology.AddNode(node)
+}
+
+func (k *kubernetesCluster) GetTopology() report.Topology {
+	k.RLock()
+	defer k.RUnlock()
+
+	return k.k8sTopology
 }
 
 func (k *kubernetesCluster) GetNode() report.TopologyNode {
+	cloudMetadata := cloud_metadata.GetCloudMetadata()
+	log.Info().Msgf("Cloud metadata: %v", cloudMetadata)
+
 	metadata := report.Metadata{
 		Timestamp:             time.Now().UTC().Format(time.RFC3339Nano),
 		NodeID:                kubernetesClusterId,
 		NodeName:              kubernetesClusterName,
 		NodeType:              report.KubernetesCluster,
 		KubernetesClusterId:   kubernetesClusterId,
 		KubernetesClusterName: kubernetesClusterName,
-		CloudProvider:         k.cloudProvider,
+		CloudProvider:         cloudMetadata.CloudProvider,
 		AgentRunning:          true,
-		CloudAccountID:        k.cloudAccountID,
+		CloudAccountID:        cloudMetadata.AccountID,
 	}
 	return report.TopologyNode{
 		Metadata: metadata,
 		Parents: &report.Parent{
-			CloudProvider: k.cloudProvider,
+			CloudProvider: cloudMetadata.CloudProvider,
 		},
 	}
 }
diff --git a/deepfence_agent/tools/apache/scope/probe/kubernetes/namespace.go b/deepfence_agent/tools/apache/scope/probe/kubernetes/namespace.go
@@ -27,7 +27,7 @@ func (ns *namespace) GetNode() report.TopologyNode {
 	return report.TopologyNode{
 		Metadata: ns.MetaNode(kubernetesClusterId+"-"+ns.Name(), report.Namespace),
 		Parents: &report.Parent{
-			CloudProvider:     cloudProviderNodeId,
+			//CloudProvider:     cloudProviderNodeId,
 			KubernetesCluster: kubernetesClusterId,
 		},
 	}

diff --git a/deepfence_agent/tools/apache/scope/probe/kubernetes/pod.go b/deepfence_agent/tools/apache/scope/probe/kubernetes/pod.go
@@ -106,7 +106,7 @@ func (p *pod) GetNode() report.TopologyNode {
 	return report.TopologyNode{
 		Metadata: metadata,
 		Parents: &report.Parent{
-			CloudProvider:     cloudProviderNodeId,
+			//CloudProvider:     cloudProviderNodeId,
 			KubernetesCluster: kubernetesClusterId,
 			Host:              hostname,
 			Namespace:         kubernetesClusterId + "-" + p.GetNamespace(),

diff --git a/deepfence_agent/tools/apache/scope/probe/kubernetes/reporter.go b/deepfence_agent/tools/apache/scope/probe/kubernetes/reporter.go
@@ -20,17 +20,16 @@ const (
 var (
 	kubernetesClusterId   string
 	kubernetesClusterName string
-	cloudProviderNodeId   string
 )
 
 // Reporter generate Reports containing Container and ContainerImage topologies
 type Reporter struct {
-	client             Client
-	probeID            string
-	probe              *probe.Probe
-	hostID             string
-	nodeName           string
-	k8sClusterTopology report.Topology
+	client                    Client
+	probeID                   string
+	probe                     *probe.Probe
+	hostID                    string
+	nodeName                  string
+	kubernetesClusterResource KubernetesClusterResource
 }
 
 // NewReporter makes a new Reporter
@@ -39,19 +38,20 @@ func NewReporter(client Client, probeID string, hostID string, probe *probe.Prob
 	kubernetesClusterName = os.Getenv(k8sClusterName)
 
 	reporter := &Reporter{
-		client:   client,
-		probeID:  probeID,
-		probe:    probe,
-		hostID:   hostID,
-		nodeName: nodeName,
+		client:                    client,
+		probeID:                   probeID,
+		probe:                     probe,
+		hostID:                    hostID,
+		nodeName:                  nodeName,
+		kubernetesClusterResource: NewKubernetesClusterResource(),
 	}
-	reporter.k8sClusterTopology = reporter.kubernetesClusterTopology()
 	//client.WatchPods(reporter.podEvent)
 	return reporter
 }
 
 // Stop unregisters controls.
 func (r *Reporter) Stop() {
+	r.kubernetesClusterResource.Stop()
 }
 
 // Name of this reporter, for metrics gathering
@@ -143,21 +143,13 @@ func (r *Reporter) Report() (report.Report, error) {
 	if err != nil {
 		return result, err
 	}
-	result.KubernetesCluster.Merge(r.k8sClusterTopology)
+	result.KubernetesCluster.Merge(r.kubernetesClusterResource.GetTopology())
 	result.Pod.Merge(podTopology)
 	result.Service.Merge(serviceTopology)
 	result.Namespace.Merge(namespaceTopology)
 	return result, nil
 }
 
-func (r *Reporter) kubernetesClusterTopology() report.Topology {
-	result := report.MakeTopology()
-	node := NewKubernetesClusterResource().GetNode()
-	cloudProviderNodeId = node.Parents.CloudProvider
-	result.AddNode(node)
-	return result
-}
-
 func (r *Reporter) serviceTopology() (report.Topology, []Service, error) {
 	var (
 		result   = report.MakeTopology()

diff --git a/deepfence_agent/tools/apache/scope/probe/kubernetes/service.go b/deepfence_agent/tools/apache/scope/probe/kubernetes/service.go
@@ -75,7 +75,7 @@ func (s *service) GetNode() report.TopologyNode {
 	return report.TopologyNode{
 		Metadata: metadata,
 		Parents: &report.Parent{
-			CloudProvider:     cloudProviderNodeId,
+			//CloudProvider:     cloudProviderNodeId,
 			KubernetesCluster: kubernetesClusterId,
 			Namespace:         kubernetesClusterId + "-" + s.GetNamespace(),
 		},

diff --git a/deepfence_frontend/apps/dashboard/api-spec.json b/deepfence_frontend/apps/dashboard/api-spec.json
@@ -13678,6 +13678,11 @@
         "required": ["node_ids"],
         "type": "object",
         "properties": {
+          "container_names": {
+            "type": "array",
+            "items": { "type": "string" },
+            "nullable": true
+          },
           "fields_filters": { "$ref": "#/components/schemas/ReportersFieldsFilters" },
           "node_ids": {
             "type": "array",
@@ -14107,10 +14112,11 @@
         }
       },
       "ModelRegistryImagesReq": {
-        "required": ["registry_id", "image_filter", "window"],
+        "required": ["registry_id", "image_filter", "image_stub_filter", "window"],
         "type": "object",
         "properties": {
           "image_filter": { "$ref": "#/components/schemas/ReportersFieldsFilters" },
+          "image_stub_filter": { "$ref": "#/components/schemas/ReportersFieldsFilters" },
           "registry_id": { "type": "string" },
           "window": { "$ref": "#/components/schemas/ModelFetchWindow" }
         }
@@ -15282,7 +15288,12 @@
         "type": "object",
         "properties": {
           "sbom_format": {
-            "enum": ["syft-json@11.0.1", "cyclonedx-json@1.5", "spdx-json@2.3"],
+            "enum": [
+              "syft-json@11.0.1",
+              "cyclonedx-json@1.5",
+              "spdx-json@2.2",
+              "spdx-json@2.3"
+            ],
             "type": "string"
           }
         }

diff --git a/deepfence_frontend/apps/dashboard/src/api/generated/models/ModelRegistryImagesReq.ts b/deepfence_frontend/apps/dashboard/src/api/generated/models/ModelRegistryImagesReq.ts
@@ -38,6 +38,12 @@ export interface ModelRegistryImagesReq {
      * @memberof ModelRegistryImagesReq
      */
     image_filter: ReportersFieldsFilters;
+    /**
+     * 
+     * @type {ReportersFieldsFilters}
+     * @memberof ModelRegistryImagesReq
+     */
+    image_stub_filter: ReportersFieldsFilters;
     /**
      * 
      * @type {string}
@@ -58,6 +64,7 @@ export interface ModelRegistryImagesReq {
 export function instanceOfModelRegistryImagesReq(value: object): boolean {
     let isInstance = true;
     isInstance = isInstance && "image_filter" in value;
+    isInstance = isInstance && "image_stub_filter" in value;
     isInstance = isInstance && "registry_id" in value;
     isInstance = isInstance && "window" in value;
 
@@ -75,6 +82,7 @@ export function ModelRegistryImagesReqFromJSONTyped(json: any, ignoreDiscriminat
     return {
 
         'image_filter': ReportersFieldsFiltersFromJSON(json['image_filter']),
+        'image_stub_filter': ReportersFieldsFiltersFromJSON(json['image_stub_filter']),
         'registry_id': json['registry_id'],
         'window': ModelFetchWindowFromJSON(json['window']),
     };
@@ -90,6 +98,7 @@ export function ModelRegistryImagesReqToJSON(value?: ModelRegistryImagesReq | nu
     return {
 
         'image_filter': ReportersFieldsFiltersToJSON(value.image_filter),
+        'image_stub_filter': ReportersFieldsFiltersToJSON(value.image_stub_filter),
         'registry_id': value.registry_id,
         'window': ModelFetchWindowToJSON(value.window),
     };

diff --git a/deepfence_frontend/apps/dashboard/src/api/generated/models/UtilsReportOptions.ts b/deepfence_frontend/apps/dashboard/src/api/generated/models/UtilsReportOptions.ts
@@ -34,6 +34,7 @@ export interface UtilsReportOptions {
 export const UtilsReportOptionsSbomFormatEnum = {
     SyftJson1101: 'syft-json@11.0.1',
     CyclonedxJson15: 'cyclonedx-json@1.5',
+    SpdxJson22: 'spdx-json@2.2',
     SpdxJson23: 'spdx-json@2.3'
 } as const;
 export type UtilsReportOptionsSbomFormatEnum = typeof UtilsReportOptionsSbomFormatEnum[keyof typeof UtilsReportOptionsSbomFormatEnum];

diff --git a/deepfence_frontend/apps/dashboard/src/components/forms/CompareScanInputModal.tsx b/deepfence_frontend/apps/dashboard/src/components/forms/CompareScanInputModal.tsx
@@ -136,7 +136,7 @@ const InputForm = ({
             valueKey="nodeId"
             onChange={(data: ISelected) => {
               setToScanData({
-                toScanTime: data.updatedAt,
+                toScanTime: data.createdAt,
                 toScanId: data.scanId,
               });
             }}
@@ -153,7 +153,7 @@ const InputForm = ({
             valueKey="nodeId"
             onChange={(data: ISelected) => {
               setToScanData({
-                toScanTime: data.updatedAt,
+                toScanTime: data.createdAt,
                 toScanId: data.scanId,
               });
             }}
@@ -183,7 +183,7 @@ export const CompareScanInputModal = ({
   showDialog: boolean;
   setShowDialog: React.Dispatch<React.SetStateAction<boolean>>;
   scanHistoryData: {
-    updatedAt: number;
+    createdAt: number;
     scanId: string;
     status: string;
   }[];
@@ -238,12 +238,12 @@ export const CompareScanInputModal = ({
             disabled={!toScanData.toScanTime}
             onClick={() => {
               const baseScan = scanHistoryData.find((data) => {
-                return data.updatedAt === compareInput.baseScanTime;
+                return data.createdAt === compareInput.baseScanTime;
               });
               setCompareInput({
                 baseScanId: baseScan?.scanId ?? '',
                 toScanId: toScanData?.toScanId ?? '',
-                baseScanTime: baseScan?.updatedAt ?? 0,
+                baseScanTime: baseScan?.createdAt ?? 0,
                 toScanTime: toScanData?.toScanTime ?? 0,
                 showScanTimeModal: false,
               });