diff --git a/deepfence_agent/plugins/YaraHunter b/deepfence_agent/plugins/YaraHunter index 930307aed7..4c76f52fc5 160000 --- a/deepfence_agent/plugins/YaraHunter +++ b/deepfence_agent/plugins/YaraHunter @@ -1 +1 @@ -Subproject commit 930307aed7cb7e684bcc45fff2f700c6213a0f7c +Subproject commit 4c76f52fc5e555e6976e6b1d515e2c1a6e42d6e8 diff --git a/deepfence_server/ingesters/agent.go b/deepfence_server/ingesters/agent.go index fd87f389bb..383607d56f 100644 --- a/deepfence_server/ingesters/agent.go +++ b/deepfence_server/ingesters/agent.go @@ -407,11 +407,13 @@ type Connection struct { leftPID int rightPID int localPort int + leftIP *string + rightIP *string } func connections2maps(connections []Connection, buf *bytes.Buffer) []map[string]interface{} { delim := ";;;" - uniqueMaps := map[string]map[string][]int{} + uniqueMaps := map[string]map[string][]interface{}{} for _, connection := range connections { buf.Reset() buf.WriteString(connection.source) @@ -420,15 +422,19 @@ func connections2maps(connections []Connection, buf *bytes.Buffer) []map[string] connectionID := buf.String() _, has := uniqueMaps[connectionID] if !has { - uniqueMaps[connectionID] = map[string][]int{ + uniqueMaps[connectionID] = map[string][]interface{}{ "left_pids": {connection.leftPID}, "right_pids": {connection.rightPID}, "local_ports": {connection.localPort}, + "left_ips": {connection.leftIP}, + "right_ips": {connection.rightIP}, } } else { uniqueMaps[connectionID]["left_pids"] = append(uniqueMaps[connectionID]["left_pids"], connection.leftPID) uniqueMaps[connectionID]["right_pids"] = append(uniqueMaps[connectionID]["right_pids"], connection.rightPID) uniqueMaps[connectionID]["local_ports"] = append(uniqueMaps[connectionID]["local_ports"], connection.localPort) + uniqueMaps[connectionID]["left_ips"] = append(uniqueMaps[connectionID]["left_ips"], connection.leftIP) + uniqueMaps[connectionID]["right_ips"] = append(uniqueMaps[connectionID]["right_ips"], connection.rightIP) } } res := make([]map[string]interface{}, 0, len(uniqueMaps)) @@ -440,12 +446,16 @@ func connections2maps(connections []Connection, buf *bytes.Buffer) []map[string] leftPIDs := v["left_pids"] rightPIDs := v["right_pids"] localPorts := v["local_ports"] - pids := make([]map[string]int, 0, len(leftPIDs)) + leftIPs := v["left_ips"] + rightIPs := v["right_ips"] + pids := make([]map[string]interface{}, 0, len(leftPIDs)) for i := range leftPIDs { - pids = append(pids, map[string]int{ + pids = append(pids, map[string]interface{}{ "left": leftPIDs[i], "right": rightPIDs[i], "local_port": localPorts[i], + "left_ip": leftIPs[i], + "right_ip": rightIPs[i], }) } internal["pids"] = pids @@ -541,15 +551,17 @@ func prepareNeo4jIngestion(rpt *report.Report, resolvers *EndpointResolversCache connections := []Connection{} localMemoization := map[string]struct{}{} for _, n := range rpt.Endpoint { - nodeIP, _ := extractIPPortFromEndpointID(n.Metadata.NodeID) + if n.Adjacency == nil || len(*n.Adjacency) == 0 { + continue + } + + nodeIP, nodePort := extractIPPortFromEndpointID(n.Metadata.NodeID) if nodeIP == localhostIP { continue } if n.Metadata.HostName == "" { if val, ok := resolvers.getHost(nodeIP, ttl); ok { n.Metadata.HostName = val - } else { - continue } } @@ -558,51 +570,52 @@ func prepareNeo4jIngestion(rpt *report.Report, resolvers *EndpointResolversCache pid = -1 } - if n.Adjacency == nil || len(*n.Adjacency) == 0 { - _, port := extractIPPortFromEndpointID(n.Metadata.NodeID) - portint, _ := strconv.Atoi(port) - // Handle inbound from internet - connections = append(connections, Connection{ - source: "in-the-internet", - destination: n.Metadata.HostName, - leftPID: 0, - rightPID: pid, - localPort: portint, - }) - } else { - for _, i := range *n.Adjacency { - if n.Metadata.NodeID != i { - ip, port := extractIPPortFromEndpointID(i) - portint, _ := strconv.Atoi(port) - // local memoization is used to skip redis access (91% reduction) - if _, has := localMemoization[ip]; has { + localPortint, _ := strconv.Atoi(nodePort) + + for _, i := range *n.Adjacency { + if n.Metadata.NodeID != i { + ip, port := extractIPPortFromEndpointID(i) + portint, _ := strconv.Atoi(port) + // local memoization is used to skip redis access (91% reduction) + if _, has := localMemoization[ip]; has { + continue + } + if host, ok := resolvers.getHost(ip, ttl); ok { + if n.Metadata.HostName == host { + localMemoization[ip] = struct{}{} continue } - if host, ok := resolvers.getHost(ip, ttl); ok { - if n.Metadata.HostName == host { - localMemoization[ip] = struct{}{} - continue - } - rightIPPID, ok := resolvers.getIPPID(ip+port, ttl) - if ok { - rightpid := extractPidFromNodeID(rightIPPID) + rightIPPID, ok := resolvers.getIPPID(ip+port, ttl) + if ok { + rightpid := extractPidFromNodeID(rightIPPID) + if n.Metadata.HostName == "" { connections = append(connections, Connection{ - source: n.Metadata.HostName, + source: "in-the-internet", destination: host, leftPID: pid, rightPID: rightpid, localPort: portint, + leftIP: &nodeIP, + }) + } else { + connections = append(connections, Connection{ + source: n.Metadata.HostName, + destination: host, + leftPID: pid, + rightPID: rightpid, + localPort: localPortint, }) } - } else { - connections = append(connections, Connection{ - source: n.Metadata.HostName, - destination: "out-the-internet", - leftPID: pid, - rightPID: 0, - localPort: portint, - }) } + } else { + connections = append(connections, Connection{ + source: n.Metadata.HostName, + destination: "out-the-internet", + leftPID: pid, + rightPID: 0, + localPort: localPortint, + rightIP: &ip, + }) } } } @@ -752,7 +765,9 @@ func (nc *neo4jIngester) PushToDBSeq(batches ReportIngestionData, session neo4j. UNWIND rpids as pids SET r.left_pids = coalesce(r.left_pids, []) + pids.left, r.right_pids = coalesce(r.right_pids, []) + pids.right, - r.local_ports = coalesce(r.local_ports, []) + pids.local_port`, + r.local_ports = coalesce(r.local_ports, []) + pids.local_port, + r.left_ips = coalesce(r.left_ips, []) + pids.left_ip, + r.right_ips = coalesce(r.right_ips, []) + pids.rigth_ip`, map[string]interface{}{"batch": batches.EndpointEdgesBatch}); err != nil { return err } diff --git a/deepfence_server/model/lookup.go b/deepfence_server/model/lookup.go index 142f13da51..c1135e2e86 100644 --- a/deepfence_server/model/lookup.go +++ b/deepfence_server/model/lookup.go @@ -49,16 +49,18 @@ type BasicNode struct { } type Connection struct { - NodeName string `json:"node_name"` - NodeID string `json:"node_id"` - Count int64 `json:"count"` + NodeName string `json:"node_name"` + NodeID string `json:"node_id"` + Count int64 `json:"count"` + IPs []interface{} `json:"ips"` } type ConnectionQueryResp struct { - FromNodeID string `json:"from_node_id"` - NodeName string `json:"node_name"` - NodeID string `json:"node_id"` - Count int64 `json:"count"` + FromNodeID string `json:"from_node_id"` + NodeName string `json:"node_name"` + NodeID string `json:"node_id"` + Count int64 `json:"count"` + IPs []interface{} `json:"ips"` } type Host struct { diff --git a/deepfence_server/reporters/lookup/lookup.go b/deepfence_server/reporters/lookup/lookup.go index 3e20f4b835..3a5ded4d84 100644 --- a/deepfence_server/reporters/lookup/lookup.go +++ b/deepfence_server/reporters/lookup/lookup.go @@ -100,6 +100,7 @@ func GetHostsReport(ctx context.Context, filter LookupFilter) ([]model.Host, err NodeName: conn.NodeName, NodeID: conn.NodeID, Count: conn.Count, + IPs: conn.IPs, }) } for _, conn := range outboundConnections { @@ -108,6 +109,7 @@ func GetHostsReport(ctx context.Context, filter LookupFilter) ([]model.Host, err NodeName: conn.NodeName, NodeID: conn.NodeID, Count: conn.Count, + IPs: conn.IPs, }) } } @@ -424,7 +426,7 @@ func getNodeConnections[T reporters.Cypherable](ctx context.Context, ids []strin query := ` MATCH (n:` + dummy.NodeType() + `)-[c:CONNECTS]-(m) WHERE n.node_id in $ids - RETURN n.node_id,m.node_id,m.node_name,sum(size(c.left_pids)),(startNode(c) = n)` + RETURN n.node_id,m.node_id,m.node_name,sum(size(c.left_pids)),(startNode(c) = n),c.left_ips,c.right_ips` r, err := tx.Run(query, map[string]interface{}{"ids": ids}) if err != nil { return inbound, outbound, err @@ -436,10 +438,17 @@ func getNodeConnections[T reporters.Cypherable](ctx context.Context, ids []strin } for _, rec := range recs { + ips := []interface{}{} + if rec.Values[5] != nil { + ips = rec.Values[5].([]interface{}) + } else if rec.Values[6] != nil { + ips = rec.Values[6].([]interface{}) + } connection := model.ConnectionQueryResp{ FromNodeID: rec.Values[0].(string), NodeID: rec.Values[1].(string), Count: rec.Values[3].(int64), + IPs: ips, } if rec.Values[2] == nil { connection.NodeName = connection.NodeID