diff --git a/logstash-core/logstash-core.gemspec b/logstash-core/logstash-core.gemspec index 03748524e51..6a5133427f5 100644 --- a/logstash-core/logstash-core.gemspec +++ b/logstash-core/logstash-core.gemspec @@ -73,4 +73,11 @@ Gem::Specification.new do |gem| gem.add_runtime_dependency "elasticsearch", "~> 5" gem.add_runtime_dependency "manticore", '~> 0.6' + + # xpack geoip database service + gem.add_development_dependency 'logstash-filter-geoip', '~> 7.1' # package hierarchy change + gem.add_dependency 'faraday' #(MIT license) + gem.add_dependency 'down', '~> 5.2.0' #(MIT license) + gem.add_dependency 'tzinfo-data' #(MIT license) + gem.add_dependency 'rufus-scheduler' #(MIT license) end diff --git a/rakelib/plugins-metadata.json b/rakelib/plugins-metadata.json index b8daea5aa4d..35f7f70266a 100644 --- a/rakelib/plugins-metadata.json +++ b/rakelib/plugins-metadata.json @@ -133,6 +133,7 @@ }, "logstash-filter-geoip": { "default-plugins": true, + "core-specs": true, "skip-list": false }, "logstash-filter-grok": { diff --git a/tools/dependencies-report/src/main/resources/licenseMapping.csv b/tools/dependencies-report/src/main/resources/licenseMapping.csv index 1504ffde392..1e475d59bfc 100644 --- a/tools/dependencies-report/src/main/resources/licenseMapping.csv +++ b/tools/dependencies-report/src/main/resources/licenseMapping.csv @@ -145,3 +145,4 @@ dependency,dependencyUrl,licenseOverride,copyright,sourceURL "unf:",https://github.com/knu/ruby-unf,BSD-2-Clause "webhdfs:",https://github.com/kzk/webhdfs,Apache-2.0 "xml-simple:",https://github.com/maik/xml-simple,BSD-2-Clause +"down",https://github.com/janko/down,MIT diff --git a/tools/dependencies-report/src/main/resources/notices/down-NOTICE.txt b/tools/dependencies-report/src/main/resources/notices/down-NOTICE.txt new file mode 100644 index 00000000000..12ad42a8be7 --- /dev/null +++ b/tools/dependencies-report/src/main/resources/notices/down-NOTICE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Janko Marohnić + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/x-pack/build.gradle b/x-pack/build.gradle index 5c978f55642..6ea1d5908ca 100644 --- a/x-pack/build.gradle +++ b/x-pack/build.gradle @@ -16,17 +16,34 @@ buildscript { } } +configurations { + geolite2 +} + dependencies { testImplementation project(':logstash-core') testImplementation 'org.assertj:assertj-core:3.8.0' testImplementation 'junit:junit:4.12' + + geolite2('org.elasticsearch:geolite2-databases:20191119') { + transitive = false + } } test { exclude '/**' } +tasks.register("unzipGeolite", Copy) { + from(zipTree(configurations.geolite2.singleFile)) { + include "GeoLite2-ASN.mmdb" + include "GeoLite2-City.mmdb" + } + into file("${projectDir}/spec/filters/geoip/vendor") +} + tasks.register("rubyTests", Test) { + dependsOn unzipGeolite inputs.files fileTree("${projectDir}/spec") inputs.files fileTree("${projectDir}/lib") inputs.files fileTree("${projectDir}/modules") diff --git a/x-pack/lib/filters/geoip/database_manager.rb b/x-pack/lib/filters/geoip/database_manager.rb new file mode 100644 index 00000000000..6147ede0838 --- /dev/null +++ b/x-pack/lib/filters/geoip/database_manager.rb @@ -0,0 +1,151 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +require "logstash/util/loggable" +require_relative "util" +require_relative "database_metadata" +require_relative "download_manager" +require "faraday" +require "json" +require "zlib" +require "stud/try" +require "down" +require "rufus/scheduler" +require "date" + +# The mission of DatabaseManager is to ensure the plugin running an up-to-date MaxMind database and +# thus users are compliant with EULA. +# DatabaseManager does a daily checking by calling an endpoint to notice a version update. +# DatabaseMetadata records the update timestamp and md5 of the database in the metadata file +# to keep track of versions and the number of days disconnects to the endpoint. +# Once a new database version release, DownloadManager downloads it, and GeoIP Filter uses it on-the-fly. +# If the last update timestamp is 25 days ago, a warning message shows in the log; +# if it was 30 days ago, the GeoIP Filter should shutdown in order to be compliant. +# There are online mode and offline mode in DatabaseManager. `online` is for automatic database update +# while `offline` is for static database path provided by users + +module LogStash module Filters module Geoip class DatabaseManager + include LogStash::Util::Loggable + include LogStash::Filters::Geoip::Util + + def initialize(geoip, database_path, database_type, vendor_path) + @vendor_path = vendor_path + @geoip = geoip + @mode = database_path.nil? ? :online : :offline + @database_type = database_type + @database_path = patch_database_path(database_path) + + if @mode == :online + logger.info "By using `online` mode, you accepted and agreed MaxMind EULA. "\ + "For more details please visit https://www.maxmind.com/en/geolite2/eula" + + setup + clean_up_database + execute_download_job + + # check database update periodically. trigger `call` method + @scheduler = Rufus::Scheduler.new({:max_work_threads => 1}) + @scheduler.every('24h', self) + else + logger.info "GeoIP plugin is in offline mode. Logstash points to static database files and will not check for update. "\ + "Keep in mind that if you are not using the database shipped with this plugin, "\ + "please go to https://www.maxmind.com/en/geolite2/eula to accept and agree the terms and conditions." + end + end + + DEFAULT_DATABASE_FILENAME = %w{ + GeoLite2-City.mmdb + GeoLite2-ASN.mmdb + }.map(&:freeze).freeze + + public + + def execute_download_job + begin + has_update, new_database_path = @download_manager.fetch_database + @database_path = new_database_path if has_update + @metadata.save_timestamp(@database_path) + has_update + rescue => e + logger.error(e.message, :cause => e.cause, :backtrace => e.backtrace) + check_age + false + end + end + + # scheduler callback + def call(job, time) + logger.debug "scheduler runs database update check" + + begin + if execute_download_job + @geoip.setup_filter(database_path) + clean_up_database + end + rescue DatabaseExpiryError => e + logger.error(e.message, :cause => e.cause, :backtrace => e.backtrace) + @geoip.terminate_filter + end + end + + def close + @scheduler.every_jobs.each(&:unschedule) if @scheduler + end + + def database_path + @database_path + end + + protected + # return a valid database path or default database path + def patch_database_path(database_path) + return database_path if file_exist?(database_path) + return database_path if database_path = get_file_path("#{DB_PREFIX}#{@database_type}.#{DB_EXT}") and file_exist?(database_path) + raise "You must specify 'database => ...' in your geoip filter (I looked for '#{database_path}')" + end + + def check_age + days_without_update = (Date.today - Time.at(@metadata.updated_at).to_date).to_i + + case + when days_without_update >= 30 + raise DatabaseExpiryError, "The MaxMind database has been used for more than 30 days. Logstash is unable to get newer version from internet. "\ + "According to EULA, GeoIP plugin needs to stop in order to be compliant. "\ + "Please check the network settings and allow Logstash accesses the internet to download the latest database, "\ + "or switch to offline mode (:database => PATH_TO_YOUR_DATABASE) to use a self-managed database which you can download from https://dev.maxmind.com/geoip/geoip2/geolite2/ " + when days_without_update >= 25 + logger.warn("The MaxMind database has been used for #{days_without_update} days without update. "\ + "Logstash will stop the GeoIP plugin in #{30 - days_without_update} days. "\ + "Please check the network settings and allow Logstash accesses the internet to download the latest database ") + else + logger.debug("The MaxMind database hasn't updated", :days_without_update => days_without_update) + end + end + + # Clean up files .mmdb, .tgz which are not mentioned in metadata and not default database + def clean_up_database + if @metadata.exist? + protected_filenames = (@metadata.database_filenames + DEFAULT_DATABASE_FILENAME).uniq + existing_filenames = ::Dir.glob(get_file_path("*.{#{DB_EXT},#{GZ_EXT}}")) + .map { |path| ::File.basename(path) } + + (existing_filenames - protected_filenames).each do |filename| + ::File.delete(get_file_path(filename)) + logger.debug("old database #{filename} is deleted") + end + end + end + + def setup + @metadata = DatabaseMetadata.new(@database_type, @vendor_path) + @metadata.save_timestamp(@database_path) unless @metadata.exist? + + @database_path = @metadata.database_path || @database_path + + @download_manager = DownloadManager.new(@database_type, @metadata, @vendor_path) + end + + class DatabaseExpiryError < StandardError + end +end end end end \ No newline at end of file diff --git a/x-pack/lib/filters/geoip/database_metadata.rb b/x-pack/lib/filters/geoip/database_metadata.rb new file mode 100644 index 00000000000..1274d6d7155 --- /dev/null +++ b/x-pack/lib/filters/geoip/database_metadata.rb @@ -0,0 +1,79 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +require "logstash/util/loggable" +require_relative "util" +require "csv" +require "date" + +module LogStash module Filters module Geoip class DatabaseMetadata + include LogStash::Util::Loggable + include LogStash::Filters::Geoip::Util + + def initialize(database_type, vendor_path) + @vendor_path = vendor_path + @metadata_path = get_file_path("metadata.csv") + @database_type = database_type + end + + public + + # csv format: database_type, update_at, gz_md5, md5, filename + def save_timestamp(database_path) + metadata = get_metadata(false) + metadata << [@database_type, Time.now.to_i, md5(get_gz_name(database_path)), md5(database_path), + ::File.basename(database_path)] + + ::CSV.open @metadata_path, 'w' do |csv| + metadata.each { |row| csv << row } + end + + logger.debug("metadata updated", :metadata => metadata) + end + + def get_all + file_exist?(@metadata_path)? ::CSV.read(@metadata_path, headers: false) : Array.new + end + + # Give rows of metadata in default database type, or empty array + def get_metadata(match_type = true) + get_all.select { |row| row[Column::DATABASE_TYPE].eql?(@database_type) == match_type } + end + + # Return database path which has valid md5 + def database_path + get_metadata.map { |metadata| [metadata, get_file_path(metadata[Column::FILENAME])] } + .select { |metadata, path| file_exist?(path) && (md5(path) == metadata[Column::MD5]) } + .map { |metadata, path| path } + .last + end + + def gz_md5 + get_metadata.map { |metadata| metadata[Column::GZ_MD5] } + .last || '' + end + + def updated_at + (get_metadata.map { |metadata| metadata[Column::UPDATE_AT] } + .last || 0).to_i + end + + # Return database related filenames in .mmdb .tgz + def database_filenames + get_all.flat_map { |metadata| [ metadata[Column::FILENAME], get_gz_name(metadata[Column::FILENAME]) ] } + end + + def exist? + file_exist?(@metadata_path) + end + + class Column + DATABASE_TYPE = 0 + UPDATE_AT = 1 + GZ_MD5 = 2 + MD5 = 3 + FILENAME = 4 + end + +end end end end \ No newline at end of file diff --git a/x-pack/lib/filters/geoip/download_manager.rb b/x-pack/lib/filters/geoip/download_manager.rb new file mode 100644 index 00000000000..351a4747135 --- /dev/null +++ b/x-pack/lib/filters/geoip/download_manager.rb @@ -0,0 +1,111 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +require_relative '../../../../lib/bootstrap/util/compress' +require "logstash/util/loggable" +require_relative "util" +require_relative "database_metadata" +require "logstash-filter-geoip_jars" +require "faraday" +require "json" +require "zlib" +require "stud/try" +require "down" +require "fileutils" + +module LogStash module Filters module Geoip class DownloadManager + include LogStash::Util::Loggable + include LogStash::Filters::Geoip::Util + + def initialize(database_type, metadata, vendor_path) + @vendor_path = vendor_path + @database_type = database_type + @metadata = metadata + end + + GEOIP_HOST = "https://geoip.elastic.co".freeze + GEOIP_PATH = "/v1/database".freeze + GEOIP_ENDPOINT = "#{GEOIP_HOST}#{GEOIP_PATH}".freeze + + public + # Check available update and download it. Unzip and validate the file. + # return [has_update, new_database_path] + def fetch_database + has_update, database_info = check_update + + if has_update + new_database_path = unzip download_database(database_info) + assert_database!(new_database_path) + return [true, new_database_path] + end + + [false, nil] + end + + def database_name + @database_name ||= "#{DB_PREFIX}#{@database_type}" + end + + def database_name_ext + @database_name_ext ||= "#{database_name}.#{DB_EXT}" + end + + private + # Call infra endpoint to get md5 of latest database and verify with metadata + # return [has_update, server db info] + def check_update + uuid = get_uuid + res = rest_client.get("#{GEOIP_ENDPOINT}?key=#{uuid}&elastic_geoip_service_tos=agree") + logger.debug("check update", :endpoint => GEOIP_ENDPOINT, :response => res.status) + + dbs = JSON.parse(res.body) + target_db = dbs.select { |db| db['name'].eql?("#{database_name}.#{GZ_EXT}") }.first + has_update = @metadata.gz_md5 != target_db['md5_hash'] + logger.info "new database version detected? #{has_update}" + + [has_update, target_db] + end + + def download_database(server_db) + Stud.try(3.times) do + new_database_zip_path = get_file_path("#{database_name}_#{Time.now.to_i}.#{GZ_EXT}") + Down.download(server_db['url'], destination: new_database_zip_path) + raise "the new download has wrong checksum" if md5(new_database_zip_path) != server_db['md5_hash'] + + logger.debug("new database downloaded in ", :path => new_database_zip_path) + new_database_zip_path + end + end + + # extract COPYRIGHT.txt, LICENSE.txt and GeoLite2-{ASN,City}.mmdb from .tgz to temp directory + def unzip(zip_path) + new_database_path = zip_path[0...-(GZ_EXT.length)] + DB_EXT + temp_dir = Stud::Temporary.pathname + + LogStash::Util::Tar.extract(zip_path, temp_dir) + logger.debug("extract database to ", :path => temp_dir) + + + FileUtils.cp(::File.join(temp_dir, database_name_ext), new_database_path) + FileUtils.cp_r(::Dir.glob(::File.join(temp_dir, "{COPYRIGHT,LICENSE}.txt")), @vendor_path) + + new_database_path + end + + # Make sure the path has usable database + def assert_database!(database_path) + raise "failed to load database #{database_path}" unless org.logstash.filters.geoip.GeoIPFilter.database_valid?(database_path) + end + + def rest_client + @client ||= Faraday.new do |conn| + conn.use Faraday::Response::RaiseError + conn.adapter :net_http + end + end + + def get_uuid + @uuid ||= ::File.read(::File.join(LogStash::SETTINGS.get("path.data"), "uuid")) + end +end end end end diff --git a/x-pack/lib/filters/geoip/util.rb b/x-pack/lib/filters/geoip/util.rb new file mode 100644 index 00000000000..64d55abd7a7 --- /dev/null +++ b/x-pack/lib/filters/geoip/util.rb @@ -0,0 +1,33 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +require "digest" + + +module LogStash module Filters + module Geoip + GZ_EXT = 'tgz'.freeze + DB_EXT = 'mmdb'.freeze + DB_PREFIX = 'GeoLite2-'.freeze + + module Util + def get_file_path(filename) + ::File.join(@vendor_path, filename) + end + + def file_exist?(path) + !path.nil? && ::File.exist?(path) && !::File.empty?(path) + end + + def md5(file_path) + file_exist?(file_path) ? Digest::MD5.hexdigest(::File.read(file_path)): "" + end + + # replace *.mmdb to *.tgz + def get_gz_name(filename) + filename[0...-(DB_EXT.length)] + GZ_EXT + end + end + end +end end \ No newline at end of file diff --git a/x-pack/spec/filters/geoip/database_manager_spec.rb b/x-pack/spec/filters/geoip/database_manager_spec.rb new file mode 100644 index 00000000000..e5f52c5a7a0 --- /dev/null +++ b/x-pack/spec/filters/geoip/database_manager_spec.rb @@ -0,0 +1,216 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +require_relative 'test_helper' +require "filters/geoip/database_manager" + +describe LogStash::Filters::Geoip do + + describe 'DatabaseManager', :aggregate_failures do + let(:mock_geoip_plugin) { double("geoip_plugin") } + let(:mock_metadata) { double("database_metadata") } + let(:mock_download_manager) { double("download_manager") } + let(:mock_scheduler) { double("scheduler") } + let(:db_manager) do + manager = LogStash::Filters::Geoip::DatabaseManager.new(mock_geoip_plugin, default_city_db_path, "City", get_vendor_path) + manager.instance_variable_set(:@metadata, mock_metadata) + manager.instance_variable_set(:@download_manager, mock_download_manager) + manager.instance_variable_set(:@scheduler, mock_scheduler) + manager + end + let(:logger) { double("Logger") } + + context "patch database" do + it "use input path" do + path = db_manager.send(:patch_database_path, default_asn_db_path) + expect(path).to eq(default_asn_db_path) + end + + it "use CC license database as default" do + path = db_manager.send(:patch_database_path, "") + expect(path).to eq(default_city_db_path) + end + + it "failed when default database is missing" do + expect(db_manager).to receive(:file_exist?).and_return(false, false) + expect { db_manager.send(:patch_database_path, "") }.to raise_error /I looked for/ + end + end + + context "md5" do + it "return md5 if file exists" do + str = db_manager.send(:md5, default_city_db_path) + expect(str).not_to eq("") + expect(str).not_to be_nil + end + + it "return empty str if file not exists" do + file = Stud::Temporary.file.path + "/invalid" + str = db_manager.send(:md5, file) + expect(str).to eq("") + end + end + + context "check age" do + it "should raise error when 30 days has passed" do + expect(mock_metadata).to receive(:updated_at).and_return((Time.now - (60 * 60 * 24 * 33)).to_i) + expect{ db_manager.send(:check_age) }.to raise_error /be compliant/ + end + + it "should give warning after 25 days" do + expect(mock_metadata).to receive(:updated_at).and_return((Time.now - (60 * 60 * 24 * 26)).to_i) + expect(mock_geoip_plugin).to receive(:terminate_filter).never + expect(LogStash::Filters::Geoip::DatabaseManager).to receive(:logger).at_least(:once).and_return(logger) + expect(logger).to receive(:warn) + expect(logger).to receive(:info) + + db_manager.send(:check_age) + end + end + + context "execute download job" do + it "should be false if no update" do + original = db_manager.instance_variable_get(:@database_path) + expect(mock_download_manager).to receive(:fetch_database).and_return([false, nil]) + allow(mock_metadata).to receive(:save_timestamp) + + expect(db_manager.send(:execute_download_job)).to be_falsey + expect(db_manager.instance_variable_get(:@database_path)).to eq(original) + end + + it "should return true if update" do + original = db_manager.instance_variable_get(:@database_path) + expect(mock_download_manager).to receive(:fetch_database).and_return([true, "NEW_PATH"]) + allow(mock_metadata).to receive(:save_timestamp) + + expect(db_manager.send(:execute_download_job)).to be_truthy + expect(db_manager.instance_variable_get(:@database_path)).not_to eq(original) + end + + it "should raise error when 30 days has passed" do + allow(mock_download_manager).to receive(:fetch_database).and_raise("boom") + expect(mock_metadata).to receive(:updated_at).and_return((Time.now - (60 * 60 * 24 * 33)).to_i) + + expect{ db_manager.send(:execute_download_job) }.to raise_error /be compliant/ + end + + + it "should return false when 25 days has passed" do + allow(mock_download_manager).to receive(:fetch_database).and_raise("boom") + + expect(mock_metadata).to receive(:updated_at).and_return((Time.now - (60 * 60 * 24 * 25)).to_i) + + expect(db_manager.send(:execute_download_job)).to be_falsey + end + end + + context "scheduler call" do + it "should call plugin termination when raise error and last update > 30 days" do + allow(mock_download_manager).to receive(:fetch_database).and_raise("boom") + expect(mock_metadata).to receive(:updated_at).and_return((Time.now - (60 * 60 * 24 * 33)).to_i) + expect(mock_geoip_plugin).to receive(:terminate_filter) + db_manager.send(:call, nil, nil) + end + + it "should not call plugin setup when database is up to date" do + allow(mock_download_manager).to receive(:fetch_database).and_return([false, nil]) + expect(mock_metadata).to receive(:save_timestamp) + allow(mock_geoip_plugin).to receive(:setup_filter).never + db_manager.send(:call, nil, nil) + end + + it "should call scheduler when has update" do + allow(db_manager).to receive(:execute_download_job).and_return(true) + allow(mock_geoip_plugin).to receive(:setup_filter).once + allow(db_manager).to receive(:clean_up_database).once + db_manager.send(:call, nil, nil) + end + end + + context "clean up database" do + let(:asn00) { get_file_path("GeoLite2-ASN_000000000.mmdb") } + let(:asn00gz) { get_file_path("GeoLite2-ASN_000000000.tgz") } + let(:city00) { get_file_path("GeoLite2-City_000000000.mmdb") } + let(:city00gz) { get_file_path("GeoLite2-City_000000000.tgz") } + let(:city44) { get_file_path("GeoLite2-City_4444444444.mmdb") } + let(:city44gz) { get_file_path("GeoLite2-City_4444444444.tgz") } + + before(:each) do + [asn00, asn00gz, city00, city00gz, city44, city44gz].each { |file_path| ::File.delete(file_path) if ::File.exist?(file_path) } + end + + it "should not delete when metadata file doesn't exist" do + expect(mock_metadata).to receive(:exist?).and_return(false) + allow(mock_geoip_plugin).to receive(:database_filenames).never + + db_manager.send(:clean_up_database) + end + + it "should delete file which is not in metadata" do + [asn00, asn00gz, city00, city00gz, city44, city44gz].each { |file_path| FileUtils.touch(file_path) } + expect(mock_metadata).to receive(:exist?).and_return(true) + expect(mock_metadata).to receive(:database_filenames).and_return(["GeoLite2-City_4444444444.mmdb"]) + + db_manager.send(:clean_up_database) + [asn00, asn00gz, city00, city00gz, city44gz].each { |file_path| expect(::File.exist?(file_path)).to be_falsey } + [default_city_db_path, default_asn_db_path, city44].each { |file_path| expect(::File.exist?(file_path)).to be_truthy } + end + + it "should keep the default database" do + expect(mock_metadata).to receive(:exist?).and_return(true) + expect(mock_metadata).to receive(:database_filenames).and_return(["GeoLite2-City_4444444444.mmdb"]) + + db_manager.send(:clean_up_database) + [default_city_db_path, default_asn_db_path].each { |file_path| expect(::File.exist?(file_path)).to be_truthy } + end + end + + context "setup metadata" do + let(:db_metadata) do + dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City", get_vendor_path) + dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path) + dbm + end + + let(:temp_metadata_path) { db_metadata.instance_variable_get(:@metadata_path) } + + before(:each) do + expect(::File.empty?(temp_metadata_path)).to be_truthy + allow(LogStash::Filters::Geoip::DatabaseMetadata).to receive(:new).and_return(db_metadata) + end + + after(:each) do + ::File.delete(second_city_db_path) if ::File.exist?(second_city_db_path) + end + + it "create metadata when file is missing" do + db_manager.send(:setup) + expect(db_manager.instance_variable_get(:@database_path)).to eql(default_city_db_path) + expect(db_metadata.database_path).to eql(default_city_db_path) + expect(::File.exist?(temp_metadata_path)).to be_truthy + expect(::File.empty?(temp_metadata_path)).to be_falsey + end + + it "manager should use database path in metadata" do + write_temp_metadata(temp_metadata_path, city2_metadata) + copy_city_database(second_city_db_name) + expect(db_metadata).to receive(:save_timestamp).never + + db_manager.send(:setup) + filename = db_manager.instance_variable_get(:@database_path).split('/').last + expect(filename).to match /#{second_city_db_name}/ + end + + it "ignore database_path in metadata if md5 does not match" do + write_temp_metadata(temp_metadata_path, ["City","","","INVALID_MD5",second_city_db_name]) + copy_city_database(second_city_db_name) + expect(db_metadata).to receive(:save_timestamp).never + + db_manager.send(:setup) + filename = db_manager.instance_variable_get(:@database_path).split('/').last + expect(filename).to match /#{default_city_db_name}/ + end + end + end +end \ No newline at end of file diff --git a/x-pack/spec/filters/geoip/database_metadata_spec.rb b/x-pack/spec/filters/geoip/database_metadata_spec.rb new file mode 100644 index 00000000000..83b78927f0b --- /dev/null +++ b/x-pack/spec/filters/geoip/database_metadata_spec.rb @@ -0,0 +1,160 @@ +# # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# # or more contributor license agreements. Licensed under the Elastic License; +# # you may not use this file except in compliance with the Elastic License. + +require_relative 'test_helper' +require "filters/geoip/database_metadata" +require "stud/temporary" + +describe LogStash::Filters::Geoip do + + describe 'DatabaseMetadata', :aggregate_failures do + let(:dbm) do + dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City", get_vendor_path) + dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path) + dbm + end + let(:temp_metadata_path) { dbm.instance_variable_get(:@metadata_path) } + let(:logger) { double("Logger") } + + context "get all" do + it "return multiple rows" do + write_temp_metadata(temp_metadata_path, city2_metadata) + + expect(dbm.get_all.size).to eq(3) + end + end + + context "get metadata" do + it "return metadata" do + write_temp_metadata(temp_metadata_path, city2_metadata) + + city = dbm.get_metadata + expect(city.size).to eq(2) + + asn = dbm.get_metadata(false) + expect(asn.size).to eq(1) + end + + it "return empty array when file is missing" do + metadata = dbm.get_metadata + expect(metadata.size).to eq(0) + end + + it "return empty array when an empty file exist" do + FileUtils.touch(temp_metadata_path) + + metadata = dbm.get_metadata + expect(metadata.size).to eq(0) + end + end + + context "save timestamp" do + before do + ::File.open(default_city_gz_path, "w") { |f| f.write "make a non empty file" } + end + + after do + delete_file(default_city_gz_path) + end + + it "write the current time" do + dbm.save_timestamp(default_city_db_path) + + metadata = dbm.get_metadata.last + expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::DATABASE_TYPE]).to eq("City") + past = metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::UPDATE_AT] + expect(Time.now.to_i - past.to_i).to be < 100 + expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::GZ_MD5]).not_to be_empty + expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::GZ_MD5]).to eq(md5(default_city_gz_path)) + expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::MD5]).to eq(default_cith_db_md5) + expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::FILENAME]).to eq(default_city_db_name) + end + end + + context "database path" do + it "return the default city database path" do + write_temp_metadata(temp_metadata_path) + + expect(dbm.database_path).to eq(default_city_db_path) + end + + it "return the last database path with valid md5" do + write_temp_metadata(temp_metadata_path, city2_metadata) + + expect(dbm.database_path).to eq(default_city_db_path) + end + + context "with ASN database type" do + let(:dbm) do + dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("ASN", get_vendor_path) + dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path) + dbm + end + + it "return the default asn database path" do + write_temp_metadata(temp_metadata_path) + + expect(dbm.database_path).to eq(default_asn_db_path) + end + end + + context "with invalid database type" do + let(:dbm) do + dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("???", get_vendor_path) + dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path) + dbm + end + + it "return nil if md5 not matched" do + write_temp_metadata(temp_metadata_path) + + expect(dbm.database_path).to be_nil + end + end + end + + context "gz md5" do + it "should give the last gz md5" do + write_temp_metadata(temp_metadata_path, ["City","","SOME_GZ_MD5","SOME_MD5",second_city_db_name]) + expect(dbm.gz_md5).to eq("SOME_GZ_MD5") + end + + it "should give empty string if metadata is empty" do + expect(dbm.gz_md5).to eq("") + end + end + + context "updated at" do + it "should give the last update timestamp" do + write_temp_metadata(temp_metadata_path, ["City","1611690807","SOME_GZ_MD5","SOME_MD5",second_city_db_name]) + expect(dbm.updated_at).to eq(1611690807) + end + + it "should give 0 if metadata is empty" do + expect(dbm.updated_at).to eq(0) + end + end + + context "database filenames" do + it "should give filename in .mmdb .tgz" do + write_temp_metadata(temp_metadata_path) + expect(dbm.database_filenames).to match_array([default_city_db_name, default_asn_db_name, + 'GeoLite2-City.tgz', 'GeoLite2-ASN.tgz']) + end + end + + context "exist" do + it "should be false because Stud create empty temp file" do + expect(dbm.exist?).to be_falsey + end + + it "should be true if temp file has content" do + ::File.open(temp_metadata_path, "w") { |f| f.write("something") } + + expect(dbm.exist?).to be_truthy + end + end + + end +end \ No newline at end of file diff --git a/x-pack/spec/filters/geoip/download_manager_spec.rb b/x-pack/spec/filters/geoip/download_manager_spec.rb new file mode 100644 index 00000000000..efd3ac0edf6 --- /dev/null +++ b/x-pack/spec/filters/geoip/download_manager_spec.rb @@ -0,0 +1,168 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +require_relative 'test_helper' +require "filters/geoip/download_manager" + +describe LogStash::Filters::Geoip do + + describe 'DownloadManager', :aggregate_failures do + let(:mock_metadata) { double("database_metadata") } + let(:download_manager) do + manager = LogStash::Filters::Geoip::DownloadManager.new( "City", mock_metadata, get_vendor_path) + manager + end + let(:logger) { double("Logger") } + + GEOIP_STAGING_HOST = "https://geoip.elastic.dev" + GEOIP_STAGING_ENDPOINT = "#{GEOIP_STAGING_HOST}#{LogStash::Filters::Geoip::DownloadManager::GEOIP_PATH}" + + before do + stub_const('LogStash::Filters::Geoip::DownloadManager::GEOIP_ENDPOINT', GEOIP_STAGING_ENDPOINT) + end + + context "rest client" do + it "can call endpoint" do + conn = download_manager.send(:rest_client) + res = conn.get("#{GEOIP_STAGING_ENDPOINT}?key=#{SecureRandom.uuid}&elastic_geoip_service_tos=agree") + expect(res.status).to eq(200) + end + + it "should raise error when endpoint response 4xx" do + conn = download_manager.send(:rest_client) + expect { conn.get("#{GEOIP_STAGING_HOST}?key=#{SecureRandom.uuid}&elastic_geoip_service_tos=agree") }.to raise_error /404/ + end + end + + context "check update" do + before(:each) do + expect(download_manager).to receive(:get_uuid).and_return(SecureRandom.uuid) + mock_resp = double("geoip_endpoint", + :body => ::File.read(::File.expand_path("./fixtures/normal_resp.json", ::File.dirname(__FILE__))), + :status => 200) + allow(download_manager).to receive_message_chain("rest_client.get").and_return(mock_resp) + end + + it "should return has_update and db info when md5 does not match" do + expect(mock_metadata).to receive(:gz_md5).and_return("") + + has_update, info = download_manager.send(:check_update) + expect(has_update).to be_truthy + expect(info).to have_key("md5_hash") + expect(info).to have_key("name") + expect(info).to have_key("provider") + expect(info).to have_key("updated") + expect(info).to have_key("url") + expect(info["name"]).to include("City") + end + + it "should return false when md5 is the same" do + expect(mock_metadata).to receive(:gz_md5).and_return("89d225ac546310b1e7979502ac9ad11c") + + has_update, info = download_manager.send(:check_update) + expect(has_update).to be_falsey + end + + it "should return true when md5 does not match" do + expect(mock_metadata).to receive(:gz_md5).and_return("bca2a8bad7e5e4013dc17343af52a841") + + has_update, info = download_manager.send(:check_update) + expect(has_update).to be_truthy + end + end + + context "download database" do + let(:db_info) do + { + "md5_hash" => md5_hash, + "name" => filename, + "provider" => "maxmind", + "updated" => 1609891257, + "url" => "https://github.com/logstash-plugins/logstash-filter-geoip/archive/master.zip" + } + end + let(:md5_hash) { SecureRandom.hex } + let(:filename) { "GeoLite2-City.tgz"} + + it "should raise error if md5 does not match" do + allow(Down).to receive(:download) + expect{ download_manager.send(:download_database, db_info) }.to raise_error /wrong checksum/ + end + + it "should download file and return zip path" do + expect(download_manager).to receive(:md5).and_return(md5_hash) + + path = download_manager.send(:download_database, db_info) + expect(path).to match /GeoLite2-City_\d+\.tgz/ + expect(::File.exist?(path)).to be_truthy + + delete_file(path) + end + end + + context "unzip" do + let(:copyright_path) { get_file_path('COPYRIGHT.txt') } + let(:license_path) { get_file_path('LICENSE.txt') } + let(:readme_path) { get_file_path('README.txt') } + + before do + file_path = ::File.expand_path("./fixtures/sample", ::File.dirname(__FILE__)) + delete_file(file_path, copyright_path, license_path, readme_path) + end + + it "should extract database and license related files" do + path = ::File.expand_path("./fixtures/sample.tgz", ::File.dirname(__FILE__)) + unzip_db_path = download_manager.send(:unzip, path) + + expect(unzip_db_path).to match /\.mmdb/ + expect(::File.exist?(unzip_db_path)).to be_truthy + expect(::File.exist?(copyright_path)).to be_truthy + expect(::File.exist?(license_path)).to be_truthy + expect(::File.exist?(readme_path)).to be_falsey + + delete_file(unzip_db_path, copyright_path, license_path) + end + end + + context "assert database" do + it "should raise error if file is invalid" do + expect{ download_manager.send(:assert_database!, "Gemfile") }.to raise_error /failed to load database/ + end + + it "should pass validation" do + expect(download_manager.send(:assert_database!, default_city_db_path)).to be_nil + end + end + + context "fetch database" do + it "should be false if no update" do + expect(download_manager).to receive(:check_update).and_return([false, {}]) + + has_update, new_database_path = download_manager.send(:fetch_database) + + expect(has_update).to be_falsey + expect(new_database_path).to be_nil + end + + it "should raise error" do + expect(download_manager).to receive(:check_update).and_return([true, {}]) + expect(download_manager).to receive(:download_database).and_raise('boom') + + expect { download_manager.send(:fetch_database) }.to raise_error + end + + it "should be true if got update" do + expect(download_manager).to receive(:check_update).and_return([true, {}]) + allow(download_manager).to receive(:download_database) + allow(download_manager).to receive(:unzip) + allow(download_manager).to receive(:assert_database!) + + has_update, new_database_path = download_manager.send(:fetch_database) + + expect(has_update).to be_truthy + end + end + + end +end \ No newline at end of file diff --git a/x-pack/spec/filters/geoip/fixtures/normal_resp.json b/x-pack/spec/filters/geoip/fixtures/normal_resp.json new file mode 100644 index 00000000000..383a32d6427 --- /dev/null +++ b/x-pack/spec/filters/geoip/fixtures/normal_resp.json @@ -0,0 +1,44 @@ +[ + { + "md5_hash": "bcfc39b5677554e091dbb19cd5cea4b0", + "name": "GeoLite2-ASN.mmdb.gz", + "provider": "maxmind", + "updated": 1615852860, + "url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-ASN.mmdb.gz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=ada6463b28177577f4981cbe5f29708d0196ed71cea0bf3c0bf8e9965c8f9fd3d184be852c4e84f24b2896d8043a466039e15b5581ba4fc7aa37a15c85c79999674a0966b28f53b0c5a8b1220b428d3c1e958f20a61e06758426b7308f1ba1966b04a2bf86a5a9f96b88c05753b429574829344d3043de1f7d2b93cade7b57d53ac6d3bcb4e6d11405f6f2e7ff8c25d813e3917177b9438f686f10bc4a006aadc6a7dde2343c9bc0017487684ad64f59bb2d0b7b73b3c817f24c91bd9afd2f36725937c8938def67d5cf6df3a7705bb40098548b55a6777ef2cd8e26c32efaa1bd0474f7f24d5e386d90e87d8a3c3aa63203a78004bccf2ad65cc97b26e94675" + }, + { + "md5_hash": "be4e335eb819af148fa4e365f176923d", + "name": "GeoLite2-ASN.tgz", + "provider": "maxmind", + "updated": 1615939277, + "url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-ASN.tgz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=8d8566fdf8167d9874966c16663a76bf8a678083c753fae0397de2eaffdb9f1d19ff36dd28bb2dc3bd9230dab5256a6d08d694574b9c50cae4b8614115ef9d3d638caf29eb18cefd7a7f0154e7baaeab4c565c828a2f050bbdbb8f5a9647d67d0748960b77846674097f76ea0d721cadda9fd99379ee604eba692c9274d238a1a3d56b7c29e236182cf5e91bae63b72d1c9a1ee7c598d7c5156683aa71a9776151bec83cb99f07f75a83483d620960fd97eca4e12c3789d72ac272912df74da1d63572609883157c6d2f115f7ab1be6b3e4503e7dd501946124f1250a299338529b8abc199afe52ff9d38904603b12b674149b85d7597e57502fda05c4b65a75" + }, + { + "md5_hash": "6cd9be41557fd4c6dd0a8609a3f96bbe", + "name": "GeoLite2-City.mmdb.gz", + "provider": "maxmind", + "updated": 1615420855, + "url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-City.mmdb.gz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=630106105d8f476a6d4e7de9fd777d8c250391ce1fbc799c7c683efeb39b319e1263948bcd326dc15f3ee0c9578f1fc95e5afe2d6b026dfac00b1fe188961df8ce3a8e5e0d71355fc0ea4d7f957af2ce8bf433210b0224d7175122ce0c1ced64dc39d2db7a979c1d173b72da58441a2358f605b92b71355cf00af4fdaa20943f21827506756b52706daaf780f173fe9f37a41fd7fc5539bbc41e79110fc4b00b37334d37179efa78c0a2ccd20ef6a5faff3baf1b5c2dfb2ef0ebb7ae4ef949f986a3cfbc8df4885476aef4ba6c06012a83418623219b48ee7ff04a41ae2ff2f421fb85fcbc04255df174647d6b9302f15441a783252c7443edfa70ef5f44068a" + }, + { + "md5_hash": "89d225ac546310b1e7979502ac9ad11c", + "name": "GeoLite2-City.tgz", + "provider": "maxmind", + "updated": 1615939277, + "url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-City.tgz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=3f5e84337ef78e8039ed391cddbcc92b0ceb3b946d4a7f60476f0633584cd3324356c9ead4bfc19f1c8776849a26b850c7e388386c5dfa8eccc2afe7e7c21d4c7fdd093cfae5c52899d9df5ffe13db6c29a0558329c8a8aecda058f9778dd23615471023fc77cc514d372d9786cbd256e27818883c1ee4b7edee75c393c89d57e94e58c2be2f9c8ee7354864b53f66d61c917eae296e071f84776e8c358218d890333fd376753a4c0f903581480629bca86d1abf3bc65efc7da30617c4847367d0ae24ba1ce0528feba3c3c3c38ecdd9a8d820d7f1a9141e30578822564c192181a97761858b9e06cc05f7db4143c89c402cbb888dcabc1f6559f4f701b79a7c" + }, + { + "md5_hash": "03bef5fb1fdc877304da3391052246dc", + "name": "GeoLite2-Country.mmdb.gz", + "provider": "maxmind", + "updated": 1615420855, + "url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-Country.mmdb.gz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=18d3266f09a8b208573fa48ca9c30cf0041b69de4eac1656cafebcf737a9f2637b0be12f9df4dd26c07bc297a4070cd0248f8874d3d03bb3fc992f7110c1c0def845f182dcc6289d5fe4faa97daf98e3bdcd2e37405bae1f04e1b293c556c352a0c574f7a52f0f0ea92bcbfb5a74542be9e651453c79a0df1f7a84f2d48d5e704ee11df9a180f9c4c76a809c6a7edab7e36b4863556d815042b9cf43fe8bb1c60f432fcae56b1779d610e8b1388addc277b0259ac595eee34227fc9884065c7aaf44c8446c4f00849d3f8dad6eba9cc7213bac33ff166dc86c344fd14da736390615bc4d00de5ba007b0b1013f46b7e81b9827d32ae9e20f779a6580f97164f9" + }, + { + "md5_hash": "c0e76a2e7e0f781028e849c2d389d8a1", + "name": "GeoLite2-Country.tgz", + "provider": "maxmind", + "updated": 1615939276, + "url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-Country.tgz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=5eaf641191c25f111afed9c569e31a5369733b3723db365b76cfbf93a7b39fd77481fe07f93fc5be2fb9ef987ef6f1c32bcb863d9d2de0e74aeece8ff568c41573c8a465e9ec5301bdc77c75b2ab369f5352f2da3f5262ae889facaf27f1685584ca49fa3bf4556ed0a92b6a4b1f1985f62378c92467d73b0c66fd1ed04cb311b903343249aed6d3ba32d7b80f0be9a08816737016038306886dcffaf141932e5fb06dfe96ff1caf8ed37f6f8128a0bdc6abf9516aeac891a791656d14f4c37b31f4c86d5dba430d92402c78d8b53dcf4ec557f0f8b6c1fb59357ae1aa7f6310289fdf16c094028570431312ea35f2c00f8cd2dcef8b98d2af5ed3ee09a7fefd" + } +] \ No newline at end of file diff --git a/x-pack/spec/filters/geoip/fixtures/sample.tgz b/x-pack/spec/filters/geoip/fixtures/sample.tgz new file mode 100644 index 00000000000..22bb93cb7a1 Binary files /dev/null and b/x-pack/spec/filters/geoip/fixtures/sample.tgz differ diff --git a/x-pack/spec/filters/geoip/test_helper.rb b/x-pack/spec/filters/geoip/test_helper.rb new file mode 100644 index 00000000000..df138a07c71 --- /dev/null +++ b/x-pack/spec/filters/geoip/test_helper.rb @@ -0,0 +1,96 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +require 'spec_helper' +require "digest" + +module GeoipHelper + def get_vendor_path + ::File.expand_path("vendor", ::File.dirname(__FILE__)) + end + + def get_file_path(filename) + ::File.join(get_vendor_path, filename) + end + + def md5(file_path) + ::File.exist?(file_path) ? Digest::MD5.hexdigest(::File.read(file_path)) : '' + end + + def default_city_db_path + get_file_path("GeoLite2-City.mmdb") + end + + def default_city_gz_path + get_file_path("GeoLite2-City.tgz") + end + + def default_asn_db_path + get_file_path("GeoLite2-ASN.mmdb") + end + + def metadata_path + get_file_path("metadata.csv") + end + + def default_city_db_name + "GeoLite2-City.mmdb" + end + + def default_asn_db_name + "GeoLite2-ASN.mmdb" + end + + def second_city_db_name + "GeoLite2-City_20200220.mmdb" + end + + def second_city_db_path + get_file_path("GeoLite2-City_20200220.mmdb") + end + + def default_cith_db_md5 + md5(default_city_db_path) + end + + def DEFAULT_ASN_DB_MD5 + md5(default_asn_db_path) + end + + + def write_temp_metadata(temp_file_path, row = nil) + now = Time.now.to_i + city = md5(default_city_db_path) + asn = md5(default_asn_db_path) + + metadata = [] + metadata << ["ASN",now,"",asn,default_asn_db_name] + metadata << ["City",now,"",city,default_city_db_name] + metadata << row if row + CSV.open temp_file_path, 'w' do |csv| + metadata.each { |row| csv << row } + end + end + + def city2_metadata + ["City",Time.now.to_i,"",md5(default_city_db_path),second_city_db_name] + end + + def copy_city_database(filename) + new_path = default_city_db_path.gsub(default_city_db_name, filename) + FileUtils.cp(default_city_db_path, new_path) + end + + def delete_file(*filepaths) + filepaths.map { |filepath| ::File.delete(filepath) if ::File.exist?(filepath) } + end + + def get_metadata_database_name + ::File.exist?(metadata_path) ? ::File.read(metadata_path).split(",").last[0..-2] : nil + end +end + +RSpec.configure do |c| + c.include GeoipHelper +end \ No newline at end of file