Skip to content

Commit

Permalink
[7.x] Geoip database service (#12675) | GeoIP clean up database after…
Browse files Browse the repository at this point in the history
… new download (#12689) | fix broken test case of term of service (#12715) | change domain and endpoint of GeoIP database service (#12727) | GeoIP database add license file (#12777)

GeoIP database service license change

Fixed: #12560
  • Loading branch information
kaisecheng authored Mar 26, 2021
1 parent d8055b8 commit 965c839
Show file tree
Hide file tree
Showing 15 changed files with 1,105 additions and 0 deletions.
7 changes: 7 additions & 0 deletions logstash-core/logstash-core.gemspec
Original file line number Diff line number Diff line change
Expand Up @@ -73,4 +73,11 @@ Gem::Specification.new do |gem|

gem.add_runtime_dependency "elasticsearch", "~> 5"
gem.add_runtime_dependency "manticore", '~> 0.6'

# xpack geoip database service
gem.add_development_dependency 'logstash-filter-geoip', '~> 7.1' # package hierarchy change
gem.add_dependency 'faraday' #(MIT license)
gem.add_dependency 'down', '~> 5.2.0' #(MIT license)
gem.add_dependency 'tzinfo-data' #(MIT license)
gem.add_dependency 'rufus-scheduler' #(MIT license)
end
1 change: 1 addition & 0 deletions rakelib/plugins-metadata.json
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@
},
"logstash-filter-geoip": {
"default-plugins": true,
"core-specs": true,
"skip-list": false
},
"logstash-filter-grok": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -145,3 +145,4 @@ dependency,dependencyUrl,licenseOverride,copyright,sourceURL
"unf:",https://github.com/knu/ruby-unf,BSD-2-Clause
"webhdfs:",https://github.com/kzk/webhdfs,Apache-2.0
"xml-simple:",https://github.com/maik/xml-simple,BSD-2-Clause
"down",https://github.com/janko/down,MIT
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
The MIT License (MIT)

Copyright (c) 2015 Janko Marohnić

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
17 changes: 17 additions & 0 deletions x-pack/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,34 @@ buildscript {
}
}

configurations {
geolite2
}

dependencies {
testImplementation project(':logstash-core')
testImplementation 'org.assertj:assertj-core:3.8.0'
testImplementation 'junit:junit:4.12'

geolite2('org.elasticsearch:geolite2-databases:20191119') {
transitive = false
}
}

test {
exclude '/**'
}

tasks.register("unzipGeolite", Copy) {
from(zipTree(configurations.geolite2.singleFile)) {
include "GeoLite2-ASN.mmdb"
include "GeoLite2-City.mmdb"
}
into file("${projectDir}/spec/filters/geoip/vendor")
}

tasks.register("rubyTests", Test) {
dependsOn unzipGeolite
inputs.files fileTree("${projectDir}/spec")
inputs.files fileTree("${projectDir}/lib")
inputs.files fileTree("${projectDir}/modules")
Expand Down
151 changes: 151 additions & 0 deletions x-pack/lib/filters/geoip/database_manager.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the Elastic License;
# you may not use this file except in compliance with the Elastic License.

require "logstash/util/loggable"
require_relative "util"
require_relative "database_metadata"
require_relative "download_manager"
require "faraday"
require "json"
require "zlib"
require "stud/try"
require "down"
require "rufus/scheduler"
require "date"

# The mission of DatabaseManager is to ensure the plugin running an up-to-date MaxMind database and
# thus users are compliant with EULA.
# DatabaseManager does a daily checking by calling an endpoint to notice a version update.
# DatabaseMetadata records the update timestamp and md5 of the database in the metadata file
# to keep track of versions and the number of days disconnects to the endpoint.
# Once a new database version release, DownloadManager downloads it, and GeoIP Filter uses it on-the-fly.
# If the last update timestamp is 25 days ago, a warning message shows in the log;
# if it was 30 days ago, the GeoIP Filter should shutdown in order to be compliant.
# There are online mode and offline mode in DatabaseManager. `online` is for automatic database update
# while `offline` is for static database path provided by users

module LogStash module Filters module Geoip class DatabaseManager
include LogStash::Util::Loggable
include LogStash::Filters::Geoip::Util

def initialize(geoip, database_path, database_type, vendor_path)
@vendor_path = vendor_path
@geoip = geoip
@mode = database_path.nil? ? :online : :offline
@database_type = database_type
@database_path = patch_database_path(database_path)

if @mode == :online
logger.info "By using `online` mode, you accepted and agreed MaxMind EULA. "\
"For more details please visit https://www.maxmind.com/en/geolite2/eula"

setup
clean_up_database
execute_download_job

# check database update periodically. trigger `call` method
@scheduler = Rufus::Scheduler.new({:max_work_threads => 1})
@scheduler.every('24h', self)
else
logger.info "GeoIP plugin is in offline mode. Logstash points to static database files and will not check for update. "\
"Keep in mind that if you are not using the database shipped with this plugin, "\
"please go to https://www.maxmind.com/en/geolite2/eula to accept and agree the terms and conditions."
end
end

DEFAULT_DATABASE_FILENAME = %w{
GeoLite2-City.mmdb
GeoLite2-ASN.mmdb
}.map(&:freeze).freeze

public

def execute_download_job
begin
has_update, new_database_path = @download_manager.fetch_database
@database_path = new_database_path if has_update
@metadata.save_timestamp(@database_path)
has_update
rescue => e
logger.error(e.message, :cause => e.cause, :backtrace => e.backtrace)
check_age
false
end
end

# scheduler callback
def call(job, time)
logger.debug "scheduler runs database update check"

begin
if execute_download_job
@geoip.setup_filter(database_path)
clean_up_database
end
rescue DatabaseExpiryError => e
logger.error(e.message, :cause => e.cause, :backtrace => e.backtrace)
@geoip.terminate_filter
end
end

def close
@scheduler.every_jobs.each(&:unschedule) if @scheduler
end

def database_path
@database_path
end

protected
# return a valid database path or default database path
def patch_database_path(database_path)
return database_path if file_exist?(database_path)
return database_path if database_path = get_file_path("#{DB_PREFIX}#{@database_type}.#{DB_EXT}") and file_exist?(database_path)
raise "You must specify 'database => ...' in your geoip filter (I looked for '#{database_path}')"
end

def check_age
days_without_update = (Date.today - Time.at(@metadata.updated_at).to_date).to_i

case
when days_without_update >= 30
raise DatabaseExpiryError, "The MaxMind database has been used for more than 30 days. Logstash is unable to get newer version from internet. "\
"According to EULA, GeoIP plugin needs to stop in order to be compliant. "\
"Please check the network settings and allow Logstash accesses the internet to download the latest database, "\
"or switch to offline mode (:database => PATH_TO_YOUR_DATABASE) to use a self-managed database which you can download from https://dev.maxmind.com/geoip/geoip2/geolite2/ "
when days_without_update >= 25
logger.warn("The MaxMind database has been used for #{days_without_update} days without update. "\
"Logstash will stop the GeoIP plugin in #{30 - days_without_update} days. "\
"Please check the network settings and allow Logstash accesses the internet to download the latest database ")
else
logger.debug("The MaxMind database hasn't updated", :days_without_update => days_without_update)
end
end

# Clean up files .mmdb, .tgz which are not mentioned in metadata and not default database
def clean_up_database
if @metadata.exist?
protected_filenames = (@metadata.database_filenames + DEFAULT_DATABASE_FILENAME).uniq
existing_filenames = ::Dir.glob(get_file_path("*.{#{DB_EXT},#{GZ_EXT}}"))
.map { |path| ::File.basename(path) }

(existing_filenames - protected_filenames).each do |filename|
::File.delete(get_file_path(filename))
logger.debug("old database #{filename} is deleted")
end
end
end

def setup
@metadata = DatabaseMetadata.new(@database_type, @vendor_path)
@metadata.save_timestamp(@database_path) unless @metadata.exist?

@database_path = @metadata.database_path || @database_path

@download_manager = DownloadManager.new(@database_type, @metadata, @vendor_path)
end

class DatabaseExpiryError < StandardError
end
end end end end
79 changes: 79 additions & 0 deletions x-pack/lib/filters/geoip/database_metadata.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the Elastic License;
# you may not use this file except in compliance with the Elastic License.

require "logstash/util/loggable"
require_relative "util"
require "csv"
require "date"

module LogStash module Filters module Geoip class DatabaseMetadata
include LogStash::Util::Loggable
include LogStash::Filters::Geoip::Util

def initialize(database_type, vendor_path)
@vendor_path = vendor_path
@metadata_path = get_file_path("metadata.csv")
@database_type = database_type
end

public

# csv format: database_type, update_at, gz_md5, md5, filename
def save_timestamp(database_path)
metadata = get_metadata(false)
metadata << [@database_type, Time.now.to_i, md5(get_gz_name(database_path)), md5(database_path),
::File.basename(database_path)]

::CSV.open @metadata_path, 'w' do |csv|
metadata.each { |row| csv << row }
end

logger.debug("metadata updated", :metadata => metadata)
end

def get_all
file_exist?(@metadata_path)? ::CSV.read(@metadata_path, headers: false) : Array.new
end

# Give rows of metadata in default database type, or empty array
def get_metadata(match_type = true)
get_all.select { |row| row[Column::DATABASE_TYPE].eql?(@database_type) == match_type }
end

# Return database path which has valid md5
def database_path
get_metadata.map { |metadata| [metadata, get_file_path(metadata[Column::FILENAME])] }
.select { |metadata, path| file_exist?(path) && (md5(path) == metadata[Column::MD5]) }
.map { |metadata, path| path }
.last
end

def gz_md5
get_metadata.map { |metadata| metadata[Column::GZ_MD5] }
.last || ''
end

def updated_at
(get_metadata.map { |metadata| metadata[Column::UPDATE_AT] }
.last || 0).to_i
end

# Return database related filenames in .mmdb .tgz
def database_filenames
get_all.flat_map { |metadata| [ metadata[Column::FILENAME], get_gz_name(metadata[Column::FILENAME]) ] }
end

def exist?
file_exist?(@metadata_path)
end

class Column
DATABASE_TYPE = 0
UPDATE_AT = 1
GZ_MD5 = 2
MD5 = 3
FILENAME = 4
end

end end end end
Loading

0 comments on commit 965c839

Please sign in to comment.