From ea014c253017a1bb1186779969f296b7056bf02f Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 15 Aug 2024 16:05:58 -0600 Subject: [PATCH] idaholab/Malcolm#542 add 'public' pseudo-segment for source/destination public IP addresses --- .../677ee170-809e-11ed-8d5b-07069f823b6f.json | 4 ++-- docs/asset-interaction-analysis.md | 2 ++ logstash/pipelines/enrichment/11_lookups.conf | 11 ++++------- logstash/pipelines/enrichment/21_netbox.conf | 10 ++++++++++ 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json b/dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json index 434969cba..13919e684 100644 --- a/dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json +++ b/dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json @@ -641,7 +641,7 @@ "updated_at": "2024-06-12T16:50:47.700Z", "version": "WzQzMiwxXQ==", "attributes": { - "title": "NetBox Enrichment - Candidate Logs", + "title": "NetBox Enrichment - Logs", "description": "", "hits": 0, "columns": [ @@ -650,7 +650,7 @@ "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"(NOT event.provider:arkime) AND ((NOT event.provider:zeek) OR event.dataset:(conn OR notice OR weird OR signatures OR software OR known*)) AND (network.direction:(internal OR inbound OR outbound))\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"tags:netbox\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ diff --git a/docs/asset-interaction-analysis.md b/docs/asset-interaction-analysis.md index db0caea89..b133e9632 100644 --- a/docs/asset-interaction-analysis.md +++ b/docs/asset-interaction-analysis.md @@ -53,6 +53,8 @@ For Malcolm's purposes, both physical devices and virtualized hosts will be stor NetBox has the concept of [sites](https://demo.netbox.dev/static/docs/core-functionality/sites-and-racks/). Sites can have overlapping IP address ranges. The site to associate with network traffic can be specified when [PCAP is uploaded](upload.md#Upload), when configuring [live analysis](live-analysis.md#LiveAnalysis), and when [configuring forwarding from Hedgehog Linux](malcolm-hedgehog-e2e-iso-install.md#Hedgehogfilebeat). If not otherwise specified, the value of the `NETBOX_DEFAULT_SITE` variable in [environment variable in `netbox-common.env`](malcolm-config.md#MalcolmConfigEnvVars) will be used for these enrichment lookups. +When NetBox enrichment is attempted for a log, the value `netbox` is automatically added to its `tags` field. + ## Compare and highlight discrepancies between NetBox inventory and observed network traffic As Malcolm cross-checks network traffic with NetBox's model (as described [above](#NetBoxEnrichment)), the resulting enrichment data (or lack thereof) can highlight devices and services observed in network traffic for which there is no corresponding entry in the list of inventoried assets. diff --git a/logstash/pipelines/enrichment/11_lookups.conf b/logstash/pipelines/enrichment/11_lookups.conf index 042285e29..337148f92 100644 --- a/logstash/pipelines/enrichment/11_lookups.conf +++ b/logstash/pipelines/enrichment/11_lookups.conf @@ -135,6 +135,8 @@ filter { mutate { id => "mutate_add_field_srcASN" add_field => { "[source][as][full]" => "AS%{[@metadata][orig_asn][asn]} %{[@metadata][orig_asn][as_org]}" } } } + mutate { id => "mutate_add_field_external_source_segment_public" + add_field => { "[source][segment][name]" => "public" } } # if reverse DNS for public IP addresses is enabled (via environment variable) do it mutate { @@ -208,6 +210,8 @@ filter { mutate { id => "mutate_add_field_dstASN" add_field => { "[destination][as][full]" => "AS%{[@metadata][resp_asn][asn]} %{[@metadata][resp_asn][as_org]}" } } } + mutate { id => "mutate_add_field_external_destination_segment_public" + add_field => { "[destination][segment][name]" => "public" } } # if reverse DNS for public IP addresses is enabled (via environment variable) do it if (![@metadata][ENV_LOGSTASH_REVERSE_DNS]) { @@ -260,19 +264,12 @@ filter { } else if ("internal_source" in [tags]) and ("external_destination" in [tags]) { mutate { id => "mutate_add_field_metadata_network_direction_outbound" add_field => { "[network][direction]" => "outbound" } } - mutate { id => "mutate_add_field_direction_outbound_destination_segment" - add_field => { "[destination][segment][name]" => "public" } } } else if ("external_source" in [tags]) and ("internal_destination" in [tags]) { mutate { id => "mutate_add_field_metadata_network_direction_inbound" add_field => { "[network][direction]" => "inbound" } } - mutate { id => "mutate_add_field_direction_inbound_source_segment" - add_field => { "[source][segment][name]" => "public" } } } else if ("external_source" in [tags]) and ("external_destination" in [tags]) { mutate { id => "mutate_add_field_metadata_network_direction_external" add_field => { "[network][direction]" => "external" } } - mutate { id => "mutate_add_field_direction_external_segments" - add_field => { "[source][segment][name]" => "public" - "[destination][segment][name]" => "public" } } } else if ("external_source" in [tags]) or ("external_destination" in [tags]) { mutate { id => "mutate_add_field_metadata_network_direction_external_one_sided" add_field => { "[network][direction]" => "external" } } diff --git a/logstash/pipelines/enrichment/21_netbox.conf b/logstash/pipelines/enrichment/21_netbox.conf index f90e6344d..ae2113eac 100644 --- a/logstash/pipelines/enrichment/21_netbox.conf +++ b/logstash/pipelines/enrichment/21_netbox.conf @@ -44,6 +44,7 @@ filter { ruby { id => "ruby_netbox_enrich_dns_ip_to_host" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_device" "source" => "[dns][resolved_ip]" @@ -70,6 +71,7 @@ filter { ruby { id => "ruby_netbox_enrich_ntlm_host_to_ip" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_device" "source" => "[source][ip]" @@ -95,6 +97,7 @@ filter { ruby { id => "ruby_netbox_enrich_ntlm_server_nb_computer_to_ip" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_device" "source" => "[destination][ip]" @@ -117,6 +120,7 @@ filter { ruby { id => "ruby_netbox_enrich_ntlm_server_dns_computer_to_ip" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_device" "source" => "[destination][ip]" @@ -145,6 +149,7 @@ filter { ruby { id => "ruby_netbox_enrich_dhcp_client_fqdn_to_ip" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_device" "source" => "[zeek][dhcp][assigned_addr]" @@ -167,6 +172,7 @@ filter { ruby { id => "ruby_netbox_enrich_dhcp_host_name_to_ip" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_device" "source" => "[zeek][dhcp][assigned_addr]" @@ -196,6 +202,7 @@ filter { ruby { id => "ruby_netbox_enrich_source_ip_segment" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "source" => "[source][ip]" "target" => "[source][segment]" @@ -214,6 +221,7 @@ filter { ruby { id => "ruby_netbox_enrich_source_ip_device" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_device" "source" => "[source][ip]" @@ -243,6 +251,7 @@ filter { ruby { id => "ruby_netbox_enrich_destination_ip_segment" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_prefix" "source" => "[destination][ip]" @@ -261,6 +270,7 @@ filter { ruby { id => "ruby_netbox_enrich_destination_ip_device" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_device" "source" => "[destination][ip]"