add option for customizing which log types get NetBox enrichment, ida…

…holab#316

* added new environment variable to logstash.env:
```
LOGSTASH_NETBOX_ENRICHMENT_DATASETS=suricata.alert,zeek.conn,zeek.known_hosts,zeek.known_services,zeek.notice,zeek.signatures,zeek.software,zeek.weird
```

config/logstash.env.example

@@ -11,6 +11,8 @@ LOGSTASH_SEVERITY_SCORING=true
 LOGSTASH_REVERSE_DNS=false
 # Whether or not Logstash will enrich network traffic metadata via NetBox API calls
 LOGSTASH_NETBOX_ENRICHMENT=false
+# Which types of logs will be enriched via NetBox (comma-separated list of provider.dataset, or the string all to enrich all logs)
+LOGSTASH_NETBOX_ENRICHMENT_DATASETS=suricata.alert,zeek.conn,zeek.known_hosts,zeek.known_services,zeek.notice,zeek.signatures,zeek.software,zeek.weird
 # Whether or not unobserved network entities in Logstash data will be used to populate NetBox
 LOGSTASH_NETBOX_AUTO_POPULATE=false
 # Caching parameters for NetBox's LogStash lookups

logstash/pipelines/enrichment/21_netbox.conf

@@ -5,20 +5,30 @@ filter {
   # Do enrichment based on NetBox lookups:
   # - source.ip -> source.device and source.segment
   # - destination.ip -> destination.device and destination.segment
-  # - TODO: source.mac -> source.device
-  # - TODO: destination.mac -> destination.device
-  # The LOGSTASH_NETBOX_ENRICHMENT environment variable is checked inside netbox_enrich.rb
+  # - source.mac -> source.device
+  # - destination.mac -> destination.device
+  # Which log types get enriched is based on the LOGSTASH_NETBOX_ENRICHMENT_DATASETS env. variable
+  # The LOGSTASH_NETBOX_ENRICHMENT env. variable is checked inside netbox_enrich.rb
   #   and will short-circuit unles this feature is enabled.
-  #
-  # Enrich zeek conn.log, notice.log, weird.log, signatures.log, software.log, known*.log and all non-zeek data sources
 
-  if (([event][provider] != "zeek") or
-      ([event][dataset] == "conn") or
-      ([event][dataset] == "notice") or
-      ([event][dataset] == "weird") or
-      ([event][dataset] == "signatures") or
-      ([event][dataset] == "software") or
-      ([event][dataset] =~ /^known/)) {
+  ruby {
+    id => "ruby_determine_netbox_suitability"
+    # $logtypes = {"suricata"=>["alert"], "zeek"=>["conn", "known_hosts", "known_services", "notice", "signatures", "software", "weird"]}
+    init => "logtypesStr = ENV['LOGSTASH_NETBOX_ENRICHMENT_DATASETS'] || 'suricata.alert,zeek.conn,zeek.known_hosts,zeek.known_services,zeek.notice,zeek.signatures,zeek.software,zeek.weird' ; logtypesArr = logtypesStr.gsub(/\s+/, '').split(','); $logtypes = logtypesArr.group_by { |logtype| logtype.split('.').first }.transform_values { |values| values.map { |v| v.split('.')[1] } }"
+    code => "
+      provider = event.get('[event][provider]').to_s
+      dataset = event.get('[event][dataset]').to_s
+      if ($logtypes.is_a?(Hash) &&
+          !$logtypes.empty? &&
+          ($logtypes.has_key?('all') ||
+           (!provider.empty? && !dataset.empty? && $logtypes.has_key?(provider) && $logtypes[provider].is_a?(Array) && $logtypes[provider].include?(dataset))))
+      then
+        event.set('[@metadata][do_netbox_enrichment]', true)
+      end
+    "
+  }
+
+  if ([@metadata][do_netbox_enrichment]) {
     if ([source][ip]) and
        (([network][direction] == "internal") or ([network][direction] == "outbound")) {
       ruby {