Use IMAP uid to retrieve new mails instead of "NOT SEEN" flag

docs/index.asciidoc

@@ -45,7 +45,9 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|Yes
 | <<plugins-{type}s-{plugin}-port>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-secure>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-sincedb_path>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-strip_attachments>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-uid_tracking>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-verify_cert>> |<<boolean,boolean>>|No
 |=======================================================================
@@ -56,15 +58,15 @@ input plugins.
 &nbsp;
 
 [id="plugins-{type}s-{plugin}-check_interval"]
-===== `check_interval` 
+===== `check_interval`
 
   * Value type is <<number,number>>
   * Default value is `300`
 
 
 
 [id="plugins-{type}s-{plugin}-content_type"]
-===== `content_type` 
+===== `content_type`
 
   * Value type is <<string,string>>
   * Default value is `"text/plain"`
@@ -73,39 +75,39 @@ For multipart messages, use the first part that has this
 content-type as the event message.
 
 [id="plugins-{type}s-{plugin}-delete"]
-===== `delete` 
+===== `delete`
 
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
 
 
 [id="plugins-{type}s-{plugin}-expunge"]
-===== `expunge` 
+===== `expunge`
 
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
 
 
 [id="plugins-{type}s-{plugin}-fetch_count"]
-===== `fetch_count` 
+===== `fetch_count`
 
   * Value type is <<number,number>>
   * Default value is `50`
 
 
 
 [id="plugins-{type}s-{plugin}-folder"]
-===== `folder` 
+===== `folder`
 
   * Value type is <<string,string>>
   * Default value is `"INBOX"`
 
 
 
 [id="plugins-{type}s-{plugin}-host"]
-===== `host` 
+===== `host`
 
   * This is a required setting.
   * Value type is <<string,string>>
@@ -114,15 +116,15 @@ content-type as the event message.
 
 
 [id="plugins-{type}s-{plugin}-lowercase_headers"]
-===== `lowercase_headers` 
+===== `lowercase_headers`
 
   * Value type is <<boolean,boolean>>
   * Default value is `true`
 
 
 
 [id="plugins-{type}s-{plugin}-password"]
-===== `password` 
+===== `password`
 
   * This is a required setting.
   * Value type is <<password,password>>
@@ -131,31 +133,54 @@ content-type as the event message.
 
 
 [id="plugins-{type}s-{plugin}-port"]
-===== `port` 
+===== `port`
 
   * Value type is <<number,number>>
   * There is no default value for this setting.
 
 
 
 [id="plugins-{type}s-{plugin}-secure"]
-===== `secure` 
+===== `secure`
 
   * Value type is <<boolean,boolean>>
   * Default value is `true`
 
 
 
+[id="plugins-{type}s-{plugin}-sincedb_path"]
+===== `sincedb_path`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Path of the sincedb database file (keeps track of the last processed IMAP
+uid) that will be written to disk. The default will write sincedb files to
+`<path.data>/plugins/inputs/imap`
+NOTE: it must be a file path and not a directory path.
+
 [id="plugins-{type}s-{plugin}-strip_attachments"]
-===== `strip_attachments` 
+===== `strip_attachments`
 
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
 
 
+[id="plugins-{type}s-{plugin}-uid_tracking"]
+===== `uid_tracking`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+IMAP uid is always stored in the file `sincedb_path` regardles of the
+`uid_tracking` setting, so you can switch between "NOT SEEN" and "UID"
+tracking. In transition from the previous plugin version, first process
+at least one mail with `uid_tracking` set to `false` to save the last
+processed IMAP uid and then switch `uid_tracking` to `true`.
+
 [id="plugins-{type}s-{plugin}-user"]
-===== `user` 
+===== `user`
 
   * This is a required setting.
   * Value type is <<string,string>>
@@ -164,7 +189,7 @@ content-type as the event message.
 
 
 [id="plugins-{type}s-{plugin}-verify_cert"]
-===== `verify_cert` 
+===== `verify_cert`
 
   * Value type is <<boolean,boolean>>
   * Default value is `true`
@@ -176,4 +201,4 @@ content-type as the event message.
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 
-:default_codec!:
+:default_codec!:

lib/logstash/inputs/imap.rb

@@ -29,11 +29,17 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
   config :delete, :validate => :boolean, :default => false
   config :expunge, :validate => :boolean, :default => false
   config :strip_attachments, :validate => :boolean, :default => false
-  
+
   # For multipart messages, use the first part that has this
   # content-type as the event message.
   config :content_type, :validate => :string, :default => "text/plain"
 
+  # Whether to use IMAP uid to track last processed message
+  config :uid_tracking, :validate => :boolean, :default => false
+
+  # Path to file with last run time metadata
+  config :sincedb_path, :validate => :string, :required => false
+
   def register
     require "net/imap" # in stdlib
     require "mail" # gem 'mail'
@@ -50,6 +56,22 @@ def register
       end
     end
 
+    # Load last processed IMAP uid from file if exists
+    if @sincedb_path.nil?
+      datapath = File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "inputs", "imap")
+      # Ensure that the filepath exists before writing, since it's deeply nested.
+      FileUtils::mkdir_p datapath
+      @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest("#{@user}_#{@host}_#{@port}_#{@folder}"))
+    end
+    if File.directory?(@sincedb_path)
+      raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"")
+    end
+    @logger.info("Using \"sincedb_path\": \"#{@sincedb_path}\"")
+    if File.exist?(@sincedb_path)
+      @uid_last_value = File.read(@sincedb_path)
+      @logger.info("Loading \"uid_last_value\": \"#{@uid_last_value}\"")
+    end
+
     @content_type_re = Regexp.new("^" + @content_type)
   end # def register
 
@@ -75,10 +97,18 @@ def check_mail(queue)
     # EOFError, OpenSSL::SSL::SSLError
     imap = connect
     imap.select(@folder)
-    ids = imap.search("NOT SEEN")
+    if @uid_tracking && @uid_last_value
+      # If there are no new messages, uid_search returns @uid_last_value
+      # because it is the last message, so we need to delete it.
+      ids = imap.uid_search(["UID", (@uid_last_value..-1)]).delete_if { |uid|
+        uid <= @uid_last_value
+      }
+    else
+      ids = imap.uid_search("NOT SEEN")
+    end
 
     ids.each_slice(@fetch_count) do |id_set|
-      items = imap.fetch(id_set, "RFC822")
+      items = imap.uid_fetch(id_set, ["RFC822", "UID"])
       items.each do |item|
         next unless item.attr.has_key?("RFC822")
         mail = Mail.read_from_string(item.attr["RFC822"])
@@ -87,16 +117,28 @@ def check_mail(queue)
         else
           queue << parse_mail(mail)
         end
+        @uid_last_value = item.attr["UID"]
+
+        # Stop mail fetching if it is requested
+        break if stop?
       end
 
+      @logger.info("Saving \"uid_last_value\": \"#{@uid_last_value}\"")
+
+      # Always save @uid_last_value so when tracking is switched from
+      # "NOT SEEN" to "UID" we will continue from first unprocessed message
+      File.write(@sincedb_path, @uid_last_value) unless @uid_last_value.nil?
       imap.store(id_set, '+FLAGS', @delete ? :Deleted : :Seen)
-
+
+      # Stop mail fetching if it is requested
+      break if stop?
     end
 
-  # Enable an 'expunge' IMAP command after the items.each loop
-    if @expunge 
-    # Force messages to be marked as "Deleted", the above may or may not be working as expected. "Seen" means nothing if you are going to
-    # delete a message after processing.
+    # Enable an 'expunge' IMAP command after the items.each loop
+    if @expunge
+      # Force messages to be marked as "Deleted", the above may or may not be
+      # working as expected. "Seen" means nothing if you are going to delete
+      # a message after processing.
       imap.store(id_set, '+FLAGS', [:Deleted])
       imap.expunge()
     end

spec/inputs/imap_spec.rb

@@ -25,7 +25,7 @@
         allow(imap).to receive(:store)
         allow(ids).to receive(:each_slice).and_return([])
 
-        allow(imap).to receive(:search).with("NOT SEEN").and_return(ids)
+        allow(imap).to receive(:uid_search).with("NOT SEEN").and_return(ids)
         allow(Net::IMAP).to receive(:new).and_return(imap)
       end
     end