From 17af43edffbbfdf63ba6e1e8eab6109d9e74dbaa Mon Sep 17 00:00:00 2001 From: Philipp Gruener Date: Wed, 30 Dec 2020 19:49:28 +0100 Subject: [PATCH] Added new maxim_db lookup and accessor for city, for the new API, which needs authorization. --- README_API_GUIDE.md | 35 ++++- .../maxmind/geolite_city_generator.rb | 2 +- .../templates/migration/geolite_city.rb | 54 ++++--- lib/geocoder/lookup.rb | 1 + lib/geocoder/lookups/maxmind_local_api.rb | 33 ++++ lib/geocoder/railtie.rb | 1 + lib/geocoder/results/maxmind_local_api.rb | 6 + lib/maxmind_database_api.rb | 143 ++++++++++++++++++ lib/tasks/maxmind_api.rake | 76 ++++++++++ 9 files changed, 326 insertions(+), 25 deletions(-) create mode 100644 lib/geocoder/lookups/maxmind_local_api.rb create mode 100644 lib/geocoder/results/maxmind_local_api.rb create mode 100644 lib/maxmind_database_api.rb create mode 100644 lib/tasks/maxmind_api.rake diff --git a/README_API_GUIDE.md b/README_API_GUIDE.md index c0cf1e739..317ad7940 100644 --- a/README_API_GUIDE.md +++ b/README_API_GUIDE.md @@ -541,7 +541,11 @@ IP Address Lookups Local IP Address Lookups ------------------------ -### MaxMind Local (`:maxmind_local`) - EXPERIMENTAL +### MaxMind Local (`:maxmind_local`) - EXPERIMENTAL - Not more working since maxmind change of 2019/12 + + +Refer for further informations: +https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases/ This lookup provides methods for geocoding IP addresses without making a call to a remote API (improves speed and availability). It works, but support is new and should not be considered production-ready. Please [report any bugs](https://github.com/alexreisner/geocoder/issues) you encounter. @@ -573,6 +577,35 @@ You can generate ActiveRecord migrations and download and import data via provid You can replace `city` with `country` in any of the above tasks, generators, and configurations. + +### MaxMind Local (`:maxmind_local_api`) - EXPERIMENTAL - Extension of original `:maxmind_local` which uses the new API + +But it's just available for the city package + +**To use a CSV file** you must import it into an SQL database. +To enable `:maxmind_local_api` configure Geocoder with the following additional settings: + + Geocoder.configure( + ip_lookup: :maxmind_local_api, + maxmind_local_api: { + package: :city, + download_api_key: 'YOUR_LICENCE_KEY', # get your free api key from https://www.maxmind.com/ + preferred_language: 'de' # set your preferred language (one of those, which is available in maxmind csv archive). Uses 'en' if nothing is defined. + } + ) + +You can generate ActiveRecord migrations and download and import data via provided rake tasks: +The new migration has **force: true** set, which will overwrite your current tables, if you were using those in the past. + + # generate migration to create tables + rails generate geocoder:maxmind:geolite_city + + # download, unpack, and import data - added another namespace for the case, the old one is needed (whyever) + rails geocoder:maxmind_api:geolite:load + +As this task really needed to be available very fast, you cannot replace `city` with `country` in this extension of the original task. + + ### GeoLite2 (`:geoip2`) This lookup provides methods for geocoding IP addresses without making a call to a remote API (improves speed and availability). It works, but support is new and should not be considered production-ready. Please [report any bugs](https://github.com/alexreisner/geocoder/issues) you encounter. diff --git a/lib/generators/geocoder/maxmind/geolite_city_generator.rb b/lib/generators/geocoder/maxmind/geolite_city_generator.rb index 985897554..373e7f638 100644 --- a/lib/generators/geocoder/maxmind/geolite_city_generator.rb +++ b/lib/generators/geocoder/maxmind/geolite_city_generator.rb @@ -11,7 +11,7 @@ class GeoliteCityGenerator < Rails::Generators::Base source_root File.expand_path('../templates', __FILE__) def copy_migration_files - migration_template "migration/geolite_city.rb", "db/migrate/geocoder_maxmind_geolite_city.rb" + migration_template "migration/geolite_city.rb", "db/migrate/geocoder_maxmind_geolite_city_new_api.rb" end # Define the next_migration_number method (necessary for the diff --git a/lib/generators/geocoder/maxmind/templates/migration/geolite_city.rb b/lib/generators/geocoder/maxmind/templates/migration/geolite_city.rb index b4068e047..e6f65f365 100644 --- a/lib/generators/geocoder/maxmind/templates/migration/geolite_city.rb +++ b/lib/generators/geocoder/maxmind/templates/migration/geolite_city.rb @@ -1,30 +1,38 @@ class GeocoderMaxmindGeoliteCity < ActiveRecord::Migration<%= migration_version %> - def self.up - create_table :maxmind_geolite_city_blocks, id: false do |t| - t.column :start_ip_num, :bigint, null: false - t.column :end_ip_num, :bigint, null: false - t.column :loc_id, :bigint, null: false - end - add_index :maxmind_geolite_city_blocks, :loc_id - add_index :maxmind_geolite_city_blocks, :start_ip_num, unique: true - add_index :maxmind_geolite_city_blocks, [:end_ip_num, :start_ip_num], unique: true, name: 'index_maxmind_geolite_city_blocks_on_end_ip_num_range' - - create_table :maxmind_geolite_city_location, id: false do |t| - t.column :loc_id, :bigint, null: false - t.string :country, null: false - t.string :region, null: false - t.string :city - t.string :postal_code, null: false + def change + create_table :maxmind_geolite_city_blocks, id: false, force: true do |t| + t.binary :start_ip_num, limit: 16, scale: 0, null: false, index: true, unique: true + t.binary :end_ip_num, limit: 16, scale: 0, null: false + t.bigint :geoname_id, index: true + t.bigint :registered_country_geoname_id + t.bigint :represented_country_geoname_id + t.boolean :is_anonymous_proxy + t.boolean :is_satellite_provider + t.string :postal_code, limit: 32 t.float :latitude t.float :longitude - t.integer :metro_code - t.integer :area_code + t.integer :accuracy_radius + end + + add_index :maxmind_geolite_city_blocks, [:end_ip_num, :start_ip_num], unique: true, name: :index_maxmind_geolite_city_blocks_on_end_ip_num_range + + create_table :maxmind_geolite_city_location, id: false, force: true do |t| + t.bigint :geoname_id, null: false + t.string :locale_code, limit: 8 # language of translation (de, en, ...) + t.string :continent_code, limit: 8 # EU, AF, AS, ... + t.string :continent_name, limit: 64 # Continent in the *locale_code*s language (Europa, Afrika, ...) + t.string :country_iso_code, limit: 8 # country code ISO: DE, BE, ... + t.string :country_name, limit: 64 # Country in the *locale_code*s language (Deutschland, Belgien, ...) + t.string :subdivision_1_iso_code, limit: 8 # ISO Code Country division: 1 (BY, HE, BW, ...) + t.string :subdivision_1_name, limit: 64 # Country division 1 in the *locale_code*s language (Bayern, Hessen, Baden-Württemberg, ...) + t.string :subdivision_2_iso_code, limit: 8 # ISO Code Country division: 2 + t.string :subdivision_2_name, limit: 64 # Country division 2 in the *locale_code*s language + t.string :city_name, limit: 64 # City name in *locale_code*s language + t.string :metro_code, limit: 32 # Metro code only us + t.string :time_zone, limit: 32 # Timezone (Europe/Berlin) + t.boolean :is_in_european_union, null: false # in EU: true/false end - add_index :maxmind_geolite_city_location, :loc_id, unique: true - end - def self.down - drop_table :maxmind_geolite_city_location - drop_table :maxmind_geolite_city_blocks + add_index :maxmind_geolite_city_location, [:geoname_id, :locale_code], unique: true, name: :index_maxmind_geolite_city_blocks_pk end end diff --git a/lib/geocoder/lookup.rb b/lib/geocoder/lookup.rb index 51b69cc28..09996dac2 100644 --- a/lib/geocoder/lookup.rb +++ b/lib/geocoder/lookup.rb @@ -67,6 +67,7 @@ def ip_services :geoip2, :maxmind, :maxmind_local, + :maxmind_local_api, :telize, :pointpin, :maxmind_geoip2, diff --git a/lib/geocoder/lookups/maxmind_local_api.rb b/lib/geocoder/lookups/maxmind_local_api.rb new file mode 100644 index 000000000..9290e2e10 --- /dev/null +++ b/lib/geocoder/lookups/maxmind_local_api.rb @@ -0,0 +1,33 @@ +require 'geocoder/lookups/maxmind_local' +require 'geocoder/results/maxmind_local_api' + +module Geocoder::Lookup + class MaxmindLocalApi < Geocoder::Lookup.get(:maxmind_local).class #::Geocoder::Lookup::MaxmindLocal + def name + "MaxMind Local - API License Protected (since 2019/12) - limited for city package" + end + + def results(query) + if (configuration[:package] || :city) == :city + addr = IPAddr.new(query.text).to_i + + q = %{ + SELECT l.country_name, + l.subdivision_1_name, + l.city_name, + b.latitude, + b.longitude + FROM maxmind_geolite_city_location AS l + LEFT JOIN maxmind_geolite_city_blocks AS b + ON l.geoname_id = b.geoname_id + AND l.locale_code = "#{configuration[:preferred_language] || 'en'}" + WHERE b.start_ip_num <= #{addr} AND #{addr} <= b.end_ip_num + } + + format_result(q, [:country_name, :region_name, :city_name, :latitude, :longitude]) + else + super(query) + end + end + end +end diff --git a/lib/geocoder/railtie.rb b/lib/geocoder/railtie.rb index 809d6f236..31b24595a 100644 --- a/lib/geocoder/railtie.rb +++ b/lib/geocoder/railtie.rb @@ -12,6 +12,7 @@ class Railtie < Rails::Railtie rake_tasks do load "tasks/geocoder.rake" load "tasks/maxmind.rake" + load "tasks/maxmind_api.rake" end end end diff --git a/lib/geocoder/results/maxmind_local_api.rb b/lib/geocoder/results/maxmind_local_api.rb new file mode 100644 index 000000000..d18935e4b --- /dev/null +++ b/lib/geocoder/results/maxmind_local_api.rb @@ -0,0 +1,6 @@ +require 'geocoder/results/maxmind_local' + +module Geocoder::Result + class MaxmindLocalApi < MaxmindLocal + end +end diff --git a/lib/maxmind_database_api.rb b/lib/maxmind_database_api.rb new file mode 100644 index 000000000..71cc82938 --- /dev/null +++ b/lib/maxmind_database_api.rb @@ -0,0 +1,143 @@ +require 'maxmind_database' + +# Maxmind API changed, so no open downloads are available anymore. Only API-protected calls are allowed. +module Geocoder + module MaxmindDatabaseApi + extend ::Geocoder::MaxmindDatabase + + class << self + def download(package, dir = "tmp") + filepath = File.expand_path(File.join(dir, archive_filename(package))) + open(filepath, 'wb') do |file| + uri = URI.parse(archive_url(package)) + Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http| # enabled use of ssl + http.request_get("#{uri.path}?#{uri.query}") do |resp| # added query parameters + puts 'downloading' + pkg_num = 0 + + resp.read_body do |segment| + pkg_num += 1 + print '.' if pkg_num % 500 == 0 + + file.write(segment) + end + + puts 'done.' + end + end + end + end + + def archive_url_path(package) + { + # geolite_country_csv: "GeoLite2-Country-CSV", # currently not supported + geolite_city_csv: 'GeoLite2-City-CSV' + # geolite_asn_csv: "GeoLite2-ASN-CSV" # currently not supported + }[package] + end + + def base_url + download_api_key = Geocoder.config[:maxmind_local_api].try(:[], :download_api_key) + raise '*maxmind_local_api -> download_api_key* is a mandatory configuration option' unless download_api_key + + "https://download.maxmind.com/app/geoip_download?license_key=#{download_api_key}&suffix=zip&edition_id=" + end + + def data_files(package, dir = 'tmp') + case package + when :geolite_city_csv + # use the last two in case multiple versions exist + city_files = Dir.glob(File.join(dir, "GeoLite2-City-CSV*/*-#{Geocoder.config[:maxmind_local_api].try(:[], :preferred_language) || 'en'}.csv")) + city_files = Dir.glob(File.join(dir, 'GeoLite2-City-CSV*/*-en.csv')) if city_files.empty? # fallback to english if preferred language isnt included in archive + + city_block_files = Dir.glob(File.join(dir, 'GeoLite2-City-CSV*/*Blocks*.csv')) + city_block_files.delete_if { |f| f =~ /IPv6/ } # skip IPv6 for now, as other datatypes or table will be necessary to improve performance + + db_tables = ['maxmind_geolite_city_blocks', 'maxmind_geolite_city_location'] + + { + 'maxmind_geolite_city_location' => city_files, + 'maxmind_geolite_city_blocks' => city_block_files + } + + when :geolite_country_csv + raise 'Currently update is not implemented. Use city instead.' + # {File.join(dir, "GeoIPCountryWhois.csv") => "maxmind_geolite_country"} + end + end + + def insert(package, dir = 'tmp') + resetted_tables = [] + + data_files(package, dir).each do |table, filepaths| + puts "Resetting table #{table}..." + ActiveRecord::Base.connection.execute("TRUNCATE TABLE #{table}") + + puts "Loading data for table #{table}" + + filepaths.each do |filepath| + puts "Inserting from file #{filepath}" + insert_into_table(table, filepath) + end + + puts 'Optimizing table' + ActiveRecord::Base.connection.execute("OPTIMIZE TABLE #{table}") + end + end + + def insert_into_table(table, filepath) + start_time = Time.now + + rows = [] + header_columns = nil + + CSV.foreach(filepath, encoding: 'utf-8') do |line| # now UTF-8! + # Each file's first record is a header; ignore it + unless header_columns + header_columns = line.to_a + next + end + + rows << line.to_a + if rows.size == 10000 + insert_rows(table, header_columns, rows) + rows = [] + print '.' + end + end + + insert_rows(table, header_columns, rows) if rows.size > 0 + + puts "\ndone (#{Time.now - start_time} seconds)" + end + + + def insert_rows(table, headers, rows) + network_col_idx = headers.index('network') + header_columns = network_col_idx ? adjust_header_columns(headers) : headers + + # adjust data from network to from-/to bigints + if network_col_idx + rows.each do |row| + addr_range = IPAddr.new(row[network_col_idx]).to_range + row[network_col_idx, 1] = [addr_range.first.to_i, addr_range.last.to_i] + end + end + + # go on with defaults + super(table, header_columns, rows) + end + + + def adjust_header_columns(header_columns) + if idx = header_columns.index('network') + new_header_columns = header_columns.dup + new_header_columns[idx, 1] = %w(start_ip_num end_ip_num) + new_header_columns + else + header_columns + end + end + end + end +end diff --git a/lib/tasks/maxmind_api.rake b/lib/tasks/maxmind_api.rake new file mode 100644 index 000000000..0b5382f46 --- /dev/null +++ b/lib/tasks/maxmind_api.rake @@ -0,0 +1,76 @@ +require 'maxmind_database_api' + +# Copied (and moved into different ns) most of maxmind.rake with minimal required adjustments. + +namespace :geocoder do + namespace :maxmind_api do + namespace :geolite do + + desc "Download and load/refresh MaxMind GeoLite City data" + task load: [:download, :extract, :insert] + + desc "Download MaxMind GeoLite City data" + task download: :environment do # critical for loading own monkey patches + p = MaxmindTaskProtected.check_for_package! + MaxmindTaskProtected.download!(p, dir: ENV['DIR'] || "tmp/") + end + + desc "Extract (unzip) MaxMind GeoLite City data" + task :extract do + p = MaxmindTaskProtected.check_for_package! + MaxmindTaskProtected.extract!(p, dir: ENV['DIR'] || "tmp/") + end + + desc "Load/refresh MaxMind GeoLite City data" + task insert: [:environment] do + p = MaxmindTaskProtected.check_for_package! + MaxmindTaskProtected.insert!(p, dir: ENV['DIR'] || "tmp/") + end + end + end +end + +module MaxmindTaskProtected + extend self + + def check_for_package! + return 'city' + # if %w[city country].include?(p = ENV['PACKAGE']) + # return p + # else + # puts "Please specify PACKAGE=city or PACKAGE=country" + # exit + # end + end + + def download!(package, options = {}) + p = "geolite_#{package}_csv".intern + Geocoder::MaxmindDatabaseApi.download(p, options[:dir]) + end + + def extract!(package, options = {}) + begin + require 'zip' + rescue LoadError + puts "Please install gem: rubyzip (>= 1.0.0)" + exit + end + require 'fileutils' + p = "geolite_#{package}_csv".intern + archive_filename = Geocoder::MaxmindDatabaseApi.archive_filename(p) + Zip::File.open(File.join(options[:dir], archive_filename)).each do |entry| + filepath = File.join(options[:dir], entry.name) + if File.exist? filepath + warn "File already exists (#{entry.name}), skipping" + else + FileUtils.mkdir_p(File.dirname(filepath)) + entry.extract(filepath) + end + end + end + + def insert!(package, options = {}) + p = "geolite_#{package}_csv".intern + Geocoder::MaxmindDatabaseApi.insert(p, options[:dir]) + end +end