Feat: Add gzip_pattern config option (#168)

Author: blangenfeld

CHANGELOG.md

@@ -1,5 +1,6 @@
 ## 3.5.0
  - Added support for including objects restored from Glacier or Glacier Deep [#199](https://github.com/logstash-plugins/logstash-input-s3/issues/199)
+ - Added `gzip_pattern` option, enabling more flexible determination of whether a file is gzipped [#165](https://github.com/logstash-plugins/logstash-input-s3/issues/165)
 
 ## 3.4.1
  - Fixed link formatting for input type (documentation)

docs/index.asciidoc

@@ -46,6 +46,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-delete>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-endpoint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-exclude_pattern>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-gzip_pattern>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-include_object_properties>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-interval>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-prefix>> |<<string,string>>|No
@@ -158,6 +159,14 @@ guaranteed to work correctly with the AWS SDK.
 
 Ruby style regexp of keys to exclude from the bucket
 
+[id="plugins-{type}s-{plugin}-gzip_pattern"]
+===== `gzip_pattern`
+
+  * Value type is <<string,string>>
+  * Default value is `"\.gz(ip)?$"`
+
+Regular expression used to determine whether an input file is in gzip format.
+
 [id="plugins-{type}s-{plugin}-additional_settings"]
 ===== `additional_settings`
 

lib/logstash/inputs/s3.rb

@@ -82,6 +82,10 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   # be present.
   config :include_object_properties, :validate => :boolean, :default => false
 
+  # Regular expression used to determine whether an input file is in gzip format.
+  # default to an expression that matches *.gz and *.gzip file extensions
+  config :gzip_pattern, :validate => :string, :default => "\.gz(ip)?$"
+
   public
   def register
     require "fileutils"
@@ -318,7 +322,7 @@ def read_gzip_file(filename, block)
 
   private
   def gzip?(filename)
-    filename.end_with?('.gz','.gzip')
+    Regexp.new(@gzip_pattern).match(filename)
   end
 
   private

spec/fixtures/compressed.log.gee.zip

@@ -443,13 +443,20 @@
       include_examples "generated events"
     end
 
-    context 'compressed with gzip extension' do
+    context 'compressed with gzip extension and using default gzip_pattern option' do
       let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5, :storage_class => 'STANDARD') }
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'compressed.log.gzip') }
 
       include_examples "generated events"
     end
 
+    context 'compressed with gzip extension and using custom gzip_pattern option' do
+      let(:config) { super.merge({ "gzip_pattern" => "gee.zip$" }) }
+      let(:log) { double(:key => 'log.gee.zip', :last_modified => Time.now - 2 * day, :content_length => 5, :storage_class => 'STANDARD') }
+      let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'compressed.log.gee.zip') }
+       include_examples "generated events"
+    end
+
     context 'plain text' do
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'uncompressed.log') }