idaholab#542 add 'public' pseudo-segment for source/destination publi…

…c IP addresses

dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json

@@ -641,7 +641,7 @@
       "updated_at": "2024-06-12T16:50:47.700Z",
       "version": "WzQzMiwxXQ==",
       "attributes": {
-        "title": "NetBox Enrichment - Candidate Logs",
+        "title": "NetBox Enrichment - Logs",
         "description": "",
         "hits": 0,
         "columns": [
@@ -650,7 +650,7 @@
         "sort": [],
         "version": 1,
         "kibanaSavedObjectMeta": {
-          "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"(NOT event.provider:arkime) AND ((NOT event.provider:zeek) OR event.dataset:(conn OR notice OR weird OR signatures OR software OR known*)) AND (network.direction:(internal OR inbound OR outbound))\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"
+          "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"tags:netbox\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"
         }
       },
       "references": [

docs/asset-interaction-analysis.md

@@ -53,6 +53,8 @@ For Malcolm's purposes, both physical devices and virtualized hosts will be stor
 
 NetBox has the concept of [sites](https://demo.netbox.dev/static/docs/core-functionality/sites-and-racks/). Sites can have overlapping IP address ranges. The site to associate with network traffic can be specified when [PCAP is uploaded](upload.md#Upload), when configuring [live analysis](live-analysis.md#LiveAnalysis), and when [configuring forwarding from Hedgehog Linux](malcolm-hedgehog-e2e-iso-install.md#Hedgehogfilebeat). If not otherwise specified, the value of the `NETBOX_DEFAULT_SITE` variable in [environment variable in `netbox-common.env`](malcolm-config.md#MalcolmConfigEnvVars) will be used for these enrichment lookups.
 
+When NetBox enrichment is attempted for a log, the value `netbox` is automatically added to its `tags` field.
+
 ## <a name="NetBoxCompare"></a>Compare and highlight discrepancies between NetBox inventory and observed network traffic
 
 As Malcolm cross-checks network traffic with NetBox's model (as described [above](#NetBoxEnrichment)), the resulting enrichment data (or lack thereof) can highlight devices and services observed in network traffic for which there is no corresponding entry in the list of inventoried assets.

logstash/pipelines/enrichment/11_lookups.conf

@@ -135,6 +135,8 @@ filter {
         mutate { id => "mutate_add_field_srcASN"
                  add_field => { "[source][as][full]" => "AS%{[@metadata][orig_asn][asn]} %{[@metadata][orig_asn][as_org]}" } }
       }
+      mutate { id => "mutate_add_field_external_source_segment_public"
+               add_field => { "[source][segment][name]" => "public" } }
 
       # if reverse DNS for public IP addresses is enabled (via environment variable) do it
       mutate {
@@ -208,6 +210,8 @@ filter {
         mutate { id => "mutate_add_field_dstASN"
                  add_field => { "[destination][as][full]" => "AS%{[@metadata][resp_asn][asn]} %{[@metadata][resp_asn][as_org]}" } }
       }
+      mutate { id => "mutate_add_field_external_destination_segment_public"
+               add_field => { "[destination][segment][name]" => "public" } }
 
       # if reverse DNS for public IP addresses is enabled (via environment variable) do it
       if (![@metadata][ENV_LOGSTASH_REVERSE_DNS]) {
@@ -260,19 +264,12 @@ filter {
   } else if ("internal_source" in [tags]) and ("external_destination" in [tags]) {
     mutate { id => "mutate_add_field_metadata_network_direction_outbound"
              add_field => { "[network][direction]" => "outbound" } }
-    mutate { id => "mutate_add_field_direction_outbound_destination_segment"
-             add_field => { "[destination][segment][name]" => "public" } }
   } else if ("external_source" in [tags]) and ("internal_destination" in [tags]) {
     mutate { id => "mutate_add_field_metadata_network_direction_inbound"
              add_field => { "[network][direction]" => "inbound" } }
-    mutate { id => "mutate_add_field_direction_inbound_source_segment"
-             add_field => { "[source][segment][name]" => "public" } }
   } else if ("external_source" in [tags]) and ("external_destination" in [tags]) {
     mutate { id => "mutate_add_field_metadata_network_direction_external"
              add_field => { "[network][direction]" => "external" } }
-    mutate { id => "mutate_add_field_direction_external_segments"
-             add_field => { "[source][segment][name]" => "public"
-                            "[destination][segment][name]" => "public" } }
   } else if ("external_source" in [tags]) or ("external_destination" in [tags]) {
     mutate { id => "mutate_add_field_metadata_network_direction_external_one_sided"
              add_field => { "[network][direction]" => "external" } }

logstash/pipelines/enrichment/21_netbox.conf

@@ -44,6 +44,7 @@ filter {
       ruby {
         id => "ruby_netbox_enrich_dns_ip_to_host"
         path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb"
+        add_tag => [ "netbox" ]
         script_params => {
           "lookup_type" => "ip_device"
           "source" => "[dns][resolved_ip]"
@@ -70,6 +71,7 @@ filter {
         ruby {
           id => "ruby_netbox_enrich_ntlm_host_to_ip"
           path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb"
+          add_tag => [ "netbox" ]
           script_params => {
             "lookup_type" => "ip_device"
             "source" => "[source][ip]"
@@ -95,6 +97,7 @@ filter {
           ruby {
             id => "ruby_netbox_enrich_ntlm_server_nb_computer_to_ip"
             path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb"
+            add_tag => [ "netbox" ]
             script_params => {
               "lookup_type" => "ip_device"
               "source" => "[destination][ip]"
@@ -117,6 +120,7 @@ filter {
           ruby {
             id => "ruby_netbox_enrich_ntlm_server_dns_computer_to_ip"
             path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb"
+            add_tag => [ "netbox" ]
             script_params => {
               "lookup_type" => "ip_device"
               "source" => "[destination][ip]"
@@ -145,6 +149,7 @@ filter {
         ruby {
           id => "ruby_netbox_enrich_dhcp_client_fqdn_to_ip"
           path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb"
+          add_tag => [ "netbox" ]
           script_params => {
             "lookup_type" => "ip_device"
             "source" => "[zeek][dhcp][assigned_addr]"
@@ -167,6 +172,7 @@ filter {
         ruby {
           id => "ruby_netbox_enrich_dhcp_host_name_to_ip"
           path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb"
+          add_tag => [ "netbox" ]
           script_params => {
             "lookup_type" => "ip_device"
             "source" => "[zeek][dhcp][assigned_addr]"
@@ -196,6 +202,7 @@ filter {
       ruby {
         id => "ruby_netbox_enrich_source_ip_segment"
         path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb"
+        add_tag => [ "netbox" ]
         script_params => {
           "source" => "[source][ip]"
           "target" => "[source][segment]"
@@ -214,6 +221,7 @@ filter {
       ruby {
         id => "ruby_netbox_enrich_source_ip_device"
         path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb"
+        add_tag => [ "netbox" ]
         script_params => {
           "lookup_type" => "ip_device"
           "source" => "[source][ip]"
@@ -243,6 +251,7 @@ filter {
       ruby {
         id => "ruby_netbox_enrich_destination_ip_segment"
         path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb"
+        add_tag => [ "netbox" ]
         script_params => {
           "lookup_type" => "ip_prefix"
           "source" => "[destination][ip]"
@@ -261,6 +270,7 @@ filter {
       ruby {
         id => "ruby_netbox_enrich_destination_ip_device"
         path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb"
+        add_tag => [ "netbox" ]
         script_params => {
           "lookup_type" => "ip_device"
           "source" => "[destination][ip]"