[7.x] Geoip database service (#12675) | GeoIP clean up database after…

… new download (#12689) | fix broken test case of term of service (#12715) | change domain and endpoint of GeoIP database service (#12727) | GeoIP database add license file (#12777)

GeoIP database service license change

Fixed: #12560

logstash-core/logstash-core.gemspec

@@ -73,4 +73,11 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency "elasticsearch", "~> 5"
   gem.add_runtime_dependency "manticore", '~> 0.6'
+
+  # xpack geoip database service
+  gem.add_development_dependency 'logstash-filter-geoip', '~> 7.1' # package hierarchy change
+  gem.add_dependency 'faraday' #(MIT license)
+  gem.add_dependency 'down', '~> 5.2.0' #(MIT license)
+  gem.add_dependency 'tzinfo-data' #(MIT license)
+  gem.add_dependency 'rufus-scheduler' #(MIT license)
 end

rakelib/plugins-metadata.json

@@ -133,6 +133,7 @@
   },
   "logstash-filter-geoip": {
     "default-plugins": true,
+    "core-specs": true,
     "skip-list": false
   },
   "logstash-filter-grok": {

tools/dependencies-report/src/main/resources/licenseMapping.csv

@@ -145,3 +145,4 @@ dependency,dependencyUrl,licenseOverride,copyright,sourceURL
 "unf:",https://github.com/knu/ruby-unf,BSD-2-Clause
 "webhdfs:",https://github.com/kzk/webhdfs,Apache-2.0
 "xml-simple:",https://github.com/maik/xml-simple,BSD-2-Clause
+"down",https://github.com/janko/down,MIT

tools/dependencies-report/src/main/resources/notices/down-NOTICE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Janko Marohnić
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

x-pack/build.gradle

@@ -16,17 +16,34 @@ buildscript {
   }
 }
 
+configurations {
+  geolite2
+}
+
 dependencies {
   testImplementation project(':logstash-core')
   testImplementation 'org.assertj:assertj-core:3.8.0'
   testImplementation 'junit:junit:4.12'
+
+  geolite2('org.elasticsearch:geolite2-databases:20191119') {
+    transitive = false
+  }
 }
 
 test {
   exclude '/**'
 }
 
+tasks.register("unzipGeolite", Copy) {
+  from(zipTree(configurations.geolite2.singleFile)) {
+    include "GeoLite2-ASN.mmdb"
+    include "GeoLite2-City.mmdb"
+  }
+  into file("${projectDir}/spec/filters/geoip/vendor")
+}
+
 tasks.register("rubyTests", Test) {
+  dependsOn unzipGeolite
   inputs.files fileTree("${projectDir}/spec")
   inputs.files fileTree("${projectDir}/lib")
   inputs.files fileTree("${projectDir}/modules")

x-pack/lib/filters/geoip/database_manager.rb

@@ -0,0 +1,151 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+require "logstash/util/loggable"
+require_relative "util"
+require_relative "database_metadata"
+require_relative "download_manager"
+require "faraday"
+require "json"
+require "zlib"
+require "stud/try"
+require "down"
+require "rufus/scheduler"
+require "date"
+
+# The mission of DatabaseManager is to ensure the plugin running an up-to-date MaxMind database and
+#   thus users are compliant with EULA.
+# DatabaseManager does a daily checking by calling an endpoint to notice a version update.
+# DatabaseMetadata records the update timestamp and md5 of the database in the metadata file
+#   to keep track of versions and the number of days disconnects to the endpoint.
+# Once a new database version release, DownloadManager downloads it, and GeoIP Filter uses it on-the-fly.
+# If the last update timestamp is 25 days ago, a warning message shows in the log;
+# if it was 30 days ago, the GeoIP Filter should shutdown in order to be compliant.
+# There are online mode and offline mode in DatabaseManager. `online` is for automatic database update
+#   while `offline` is for static database path provided by users
+
+module LogStash module Filters module Geoip class DatabaseManager
+  include LogStash::Util::Loggable
+  include LogStash::Filters::Geoip::Util
+
+  def initialize(geoip, database_path, database_type, vendor_path)
+    @vendor_path = vendor_path
+    @geoip = geoip
+    @mode = database_path.nil? ? :online : :offline
+    @database_type = database_type
+    @database_path = patch_database_path(database_path)
+
+    if @mode == :online
+      logger.info "By using `online` mode, you accepted and agreed MaxMind EULA. "\
+                  "For more details please visit https://www.maxmind.com/en/geolite2/eula"
+
+      setup
+      clean_up_database
+      execute_download_job
+
+      # check database update periodically. trigger `call` method
+      @scheduler = Rufus::Scheduler.new({:max_work_threads => 1})
+      @scheduler.every('24h', self)
+    else
+      logger.info "GeoIP plugin is in offline mode. Logstash points to static database files and will not check for update. "\
+                  "Keep in mind that if you are not using the database shipped with this plugin, "\
+                  "please go to https://www.maxmind.com/en/geolite2/eula to accept and agree the terms and conditions."
+    end
+  end
+
+  DEFAULT_DATABASE_FILENAME = %w{
+    GeoLite2-City.mmdb
+    GeoLite2-ASN.mmdb
+  }.map(&:freeze).freeze
+
+  public
+
+  def execute_download_job
+    begin
+      has_update, new_database_path = @download_manager.fetch_database
+      @database_path = new_database_path if has_update
+      @metadata.save_timestamp(@database_path)
+      has_update
+    rescue => e
+      logger.error(e.message, :cause => e.cause, :backtrace => e.backtrace)
+      check_age
+      false
+    end
+  end
+
+  # scheduler callback
+  def call(job, time)
+    logger.debug "scheduler runs database update check"
+
+    begin
+      if execute_download_job
+        @geoip.setup_filter(database_path)
+        clean_up_database
+      end
+    rescue DatabaseExpiryError => e
+      logger.error(e.message, :cause => e.cause, :backtrace => e.backtrace)
+      @geoip.terminate_filter
+    end
+  end
+
+  def close
+    @scheduler.every_jobs.each(&:unschedule) if @scheduler
+  end
+
+  def database_path
+    @database_path
+  end
+
+  protected
+  # return a valid database path or default database path
+  def patch_database_path(database_path)
+    return database_path if file_exist?(database_path)
+    return database_path if database_path = get_file_path("#{DB_PREFIX}#{@database_type}.#{DB_EXT}") and file_exist?(database_path)
+    raise "You must specify 'database => ...' in your geoip filter (I looked for '#{database_path}')"
+  end
+
+  def check_age
+    days_without_update = (Date.today - Time.at(@metadata.updated_at).to_date).to_i
+
+    case
+    when days_without_update >= 30
+      raise DatabaseExpiryError, "The MaxMind database has been used for more than 30 days. Logstash is unable to get newer version from internet. "\
+      "According to EULA, GeoIP plugin needs to stop in order to be compliant. "\
+      "Please check the network settings and allow Logstash accesses the internet to download the latest database, "\
+      "or switch to offline mode (:database => PATH_TO_YOUR_DATABASE) to use a self-managed database which you can download from https://dev.maxmind.com/geoip/geoip2/geolite2/ "
+    when days_without_update >= 25
+      logger.warn("The MaxMind database has been used for #{days_without_update} days without update. "\
+      "Logstash will stop the GeoIP plugin in #{30 - days_without_update} days. "\
+      "Please check the network settings and allow Logstash accesses the internet to download the latest database ")
+    else
+      logger.debug("The MaxMind database hasn't updated", :days_without_update => days_without_update)
+    end
+  end
+
+  # Clean up files .mmdb, .tgz which are not mentioned in metadata and not default database
+  def clean_up_database
+    if @metadata.exist?
+      protected_filenames = (@metadata.database_filenames + DEFAULT_DATABASE_FILENAME).uniq
+      existing_filenames = ::Dir.glob(get_file_path("*.{#{DB_EXT},#{GZ_EXT}}"))
+                                .map { |path| ::File.basename(path) }
+
+      (existing_filenames - protected_filenames).each do |filename|
+        ::File.delete(get_file_path(filename))
+        logger.debug("old database #{filename} is deleted")
+      end
+    end
+  end
+
+  def setup
+    @metadata = DatabaseMetadata.new(@database_type, @vendor_path)
+    @metadata.save_timestamp(@database_path) unless @metadata.exist?
+
+    @database_path = @metadata.database_path || @database_path
+
+    @download_manager = DownloadManager.new(@database_type, @metadata, @vendor_path)
+  end
+
+  class DatabaseExpiryError < StandardError
+  end
+end end end end

x-pack/lib/filters/geoip/database_metadata.rb

@@ -0,0 +1,79 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+require "logstash/util/loggable"
+require_relative "util"
+require "csv"
+require "date"
+
+module LogStash module Filters module Geoip class DatabaseMetadata
+  include LogStash::Util::Loggable
+  include LogStash::Filters::Geoip::Util
+
+  def initialize(database_type, vendor_path)
+    @vendor_path = vendor_path
+    @metadata_path = get_file_path("metadata.csv")
+    @database_type = database_type
+  end
+
+  public
+
+  # csv format: database_type, update_at, gz_md5, md5, filename
+  def save_timestamp(database_path)
+    metadata = get_metadata(false)
+    metadata << [@database_type, Time.now.to_i, md5(get_gz_name(database_path)), md5(database_path),
+                 ::File.basename(database_path)]
+
+    ::CSV.open @metadata_path, 'w' do |csv|
+      metadata.each { |row| csv << row }
+    end
+
+    logger.debug("metadata updated", :metadata => metadata)
+  end
+
+  def get_all
+    file_exist?(@metadata_path)? ::CSV.read(@metadata_path, headers: false) : Array.new
+  end
+
+  # Give rows of metadata in default database type, or empty array
+  def get_metadata(match_type = true)
+    get_all.select { |row| row[Column::DATABASE_TYPE].eql?(@database_type) == match_type }
+  end
+
+  # Return database path which has valid md5
+  def database_path
+    get_metadata.map { |metadata| [metadata, get_file_path(metadata[Column::FILENAME])] }
+                .select { |metadata, path| file_exist?(path) && (md5(path) == metadata[Column::MD5]) }
+                .map { |metadata, path| path }
+                .last
+  end
+
+  def gz_md5
+    get_metadata.map { |metadata| metadata[Column::GZ_MD5] }
+                .last || ''
+  end
+
+  def updated_at
+    (get_metadata.map { |metadata| metadata[Column::UPDATE_AT] }
+                 .last || 0).to_i
+  end
+
+  # Return database related filenames in .mmdb .tgz
+  def database_filenames
+    get_all.flat_map { |metadata| [ metadata[Column::FILENAME], get_gz_name(metadata[Column::FILENAME]) ] }
+  end
+
+  def exist?
+    file_exist?(@metadata_path)
+  end
+
+  class Column
+    DATABASE_TYPE = 0
+    UPDATE_AT     = 1
+    GZ_MD5        = 2
+    MD5           = 3
+    FILENAME      = 4
+  end
+
+end end end end