idaholab#234, update logstash-oss container to 8.9.0 and handle loadi…

…ng large YAML documents in mac_lookup.rb and netbox_enrich.rb

Dockerfiles/logstash.Dockerfile

@@ -1,4 +1,4 @@
-FROM opensearchproject/logstash-oss-with-opensearch-output-plugin:8.6.1
+FROM docker.elastic.co/logstash/logstash-oss:8.9.0
 
 LABEL maintainer="malcolm@inl.gov"
 LABEL org.opencontainers.image.authors='malcolm@inl.gov'
@@ -66,13 +66,14 @@ RUN set -x && \
     echo "gem 'deep_merge'" >> /usr/share/logstash/Gemfile && \
     echo "gem 'fuzzy-string-match'" >> /usr/share/logstash/Gemfile && \
     echo "gem 'stringex'" >> /usr/share/logstash/Gemfile && \
+    echo "gem 'psych'" >> /usr/share/logstash/Gemfile && \
     /usr/share/logstash/bin/ruby -S bundle install && \
     logstash-plugin install --preserve logstash-filter-translate logstash-filter-cidr logstash-filter-dns \
                                        logstash-filter-json logstash-filter-prune logstash-filter-http \
                                        logstash-filter-grok logstash-filter-geoip logstash-filter-uuid \
                                        logstash-filter-kv logstash-filter-mutate logstash-filter-dissect \
                                        logstash-filter-fingerprint logstash-filter-useragent \
-                                       logstash-input-beats logstash-output-elasticsearch && \
+                                       logstash-input-beats logstash-output-elasticsearch logstash-output-opensearch && \
     apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages autoremove && \
         apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/bin/jruby \

logstash/ruby/mac_lookup.rb

@@ -3,11 +3,13 @@ def concurrency
 end
 
 def register(params)
+  require 'psych'
+
   @source = params["source"]
   @target = params["target"]
   if File.exist?(params["map_path"])
     @macarray = Array.new
-    YAML.safe_load(File.read(params["map_path"])).each do |mac|
+    psych_load_yaml(params["map_path"]).each do |mac|
       @macarray.push([mac_string_to_integer(mac['low']), mac_string_to_integer(mac['high']), mac['name']])
     end
     # Array.bsearch only works on a sorted array
@@ -57,6 +59,18 @@ def mac_string_to_integer(string)
   string.tr('.:-','').to_i(16)
 end
 
+def psych_load_yaml(filename)
+  parser = Psych::Parser.new(Psych::TreeBuilder.new)
+  parser.code_point_limit = 64*1024*1024
+  parser.parse(IO.read(filename, :mode => 'r:bom|utf-8'))
+  yaml_obj = Psych::Visitors::ToRuby.create().accept(parser.handler.root)
+  if yaml_obj.is_a?(Array) && (yaml_obj.length() == 1)
+    yaml_obj.first
+  else
+    yaml_obj
+  end
+end
+
 ###############################################################################
 # tests
 

logstash/ruby/netbox_enrich.rb

@@ -9,6 +9,7 @@ def register(params)
   require 'ipaddr'
   require 'json'
   require 'lru_redux'
+  require 'psych'
   require 'stringex_lite'
 
   # global enable/disable for this plugin based on environment variable(s)
@@ -116,7 +117,7 @@ def register(params)
   _vendor_oui_map_path = params.fetch("vendor_oui_map_path", "/etc/vendor_macs.yaml")
   if File.exist?(_vendor_oui_map_path)
     @macarray = Array.new
-    YAML.safe_load(File.read(_vendor_oui_map_path)).each do |mac|
+    psych_load_yaml(_vendor_oui_map_path).each do |mac|
       @macarray.push([mac_string_to_integer(mac['low']), mac_string_to_integer(mac['high']), mac['name']])
     end
     # Array.bsearch only works on a sorted array
@@ -129,7 +130,7 @@ def register(params)
   _vm_oui_map_path = params.fetch("vm_oui_map_path", "/etc/vm_macs.yaml")
   if File.exist?(_vm_oui_map_path)
     @vm_namesarray = Set.new
-    YAML.safe_load(File.read(_vm_oui_map_path)).each do |mac|
+    psych_load_yaml(_vm_oui_map_path).each do |mac|
       @vm_namesarray.add(mac['name'].to_s.downcase)
     end
   else
@@ -724,6 +725,18 @@ def mac_string_to_integer(string)
   string.tr('.:-','').to_i(16)
 end
 
+def psych_load_yaml(filename)
+  parser = Psych::Parser.new(Psych::TreeBuilder.new)
+  parser.code_point_limit = 64*1024*1024
+  parser.parse(IO.read(filename, :mode => 'r:bom|utf-8'))
+  yaml_obj = Psych::Visitors::ToRuby.create().accept(parser.handler.root)
+  if yaml_obj.is_a?(Array) && (yaml_obj.length() == 1)
+    yaml_obj.first
+  else
+    yaml_obj
+  end
+end
+
 def collect_values(hashes)
   # https://stackoverflow.com/q/5490952
   hashes.reduce({}){ |h, pairs| pairs.each { |k,v| (h[k] ||= []) << v}; h }