diff --git a/README.md b/README.md index 722fa16..362f9fc 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,32 @@ # SeventeenMon -SeventeenMon simply help you find location by IP address. Data is totally based on [17MON.CN](http://tool.17mon.cn/). +SeventeenMon simply help you find location by IP address. Data is totally based on [IPIP.NET](https://www.ipip.net/product/client.html). +Version >= 2.0 only support ipdb format. ## Compatibility -I have tested it on Ruby >= 1.9.3. Other versions are not tested but should work on well. Please contact me if not. +I have tested it on Ruby 2.1.10, 2.4.6, 2.6.2. Other versions are not tested but should work on well. Please contact me if not. ## Installation Add this line to your application's Gemfile: - gem 'seventeen_mon', git: "git@github.com:mycolorway/SeventeenMon.git" + gem 'seventeen_mon', git: "https://github.com/BlindingDark/SeventeenMon" And then execute: $ bundle -Or you can install simply by +This project is forked from [mycolorway/SeventeenMon](https://github.com/mycolorway/SeventeenMon). +Install by gem will be used the mycolorway's **old version**. - $ gem install seventeen_mon +I am considering releasing a new gem package, but I didn't think of a good name. Please contact me if you have some good idea, thanks. ## Usage ### In Ruby + ```(ruby) SM.find_by_ip "119.75.216.20" # => {:country=>"中国", :province=>"北京", :city=>"北京"} @@ -47,9 +50,14 @@ Province: 浙江 City: 杭州 ``` +## TODO + +* [X] More metadata info +* [ ] Support paid version format + ## Contributing -1. Fork it ( http://github.com//seventeen_mom/fork ) +1. Fork it ( http://github.com/BlindingDark/SeventeenMon/fork ) 2. Create your feature branch (`git checkout -b my-new-feature`) 3. Commit your changes (`git commit -am 'Add some feature'`) 4. Push to the branch (`git push origin my-new-feature`) @@ -57,4 +65,4 @@ City: 杭州 ## Thanks -[高春辉 Paul Gao](http://tool.17mon.cn/) - for his awesome data. \ No newline at end of file +[高春辉 Paul Gao](https://www.ipip.net) - for his awesome data. diff --git a/lib/data/17monipdb.dat b/lib/data/17monipdb.dat deleted file mode 100755 index 5cd4760..0000000 Binary files a/lib/data/17monipdb.dat and /dev/null differ diff --git a/lib/data/ipip.ipdb b/lib/data/ipip.ipdb new file mode 100644 index 0000000..911e1ac Binary files /dev/null and b/lib/data/ipip.ipdb differ diff --git a/lib/seventeen_mon.rb b/lib/seventeen_mon.rb index d60eaf3..cd41409 100644 --- a/lib/seventeen_mon.rb +++ b/lib/seventeen_mon.rb @@ -1,20 +1,25 @@ -# coding: utf-8 - module SeventeenMon require "socket" require "ipaddr" require "seventeen_mon/version" require "seventeen_mon/ipdb" + require "seventeen_mon/ipdb_metadata" require "seventeen_mon/ip" - def self.find_by_ip(_ip) - IP.new(ip: _ip).find - end + class << self + def find_by_ip(ip, lang = "CN") + IP.new(ip: ip, language: lang).find + end + + def find_by_address(address, lang = "CN") + prot, addr = address.split("://") + IP.new(address: addr, protocol: prot, language: lang).find + end - def self.find_by_address(_address) - prot, addr = _address.split("://") - IP.new(address: addr, protocol: prot).find + def metadata + IPDBMetadata.new(IPDB.instance.metadata) + end end end -SM = SeventeenMon \ No newline at end of file +SM = SeventeenMon diff --git a/lib/seventeen_mon/ip.rb b/lib/seventeen_mon/ip.rb index d7fcb2a..70bbecc 100644 --- a/lib/seventeen_mon/ip.rb +++ b/lib/seventeen_mon/ip.rb @@ -1,6 +1,6 @@ module SeventeenMon class IP - attr_reader :ip + attr_reader :ip, :ip_addr # Initialize IP object # @@ -12,52 +12,33 @@ class IP # self # def initialize(params = {}) - @ip = params[:ip] || - Socket.getaddrinfo(params[:address], params[:protocol])[0][3] + @ip = params[:ip] || Socket.getaddrinfo(params[:address], params[:protocol])[0][3] + @ip_addr = IPAddr.new(ip) + @language = params[:language] end - def four_number - @four_number ||= begin - fn = ip.split(".").map(&:to_i) - raise "ip is no valid" if fn.length != 4 || fn.any?{ |d| d < 0 || d > 255} - fn - end - end - - def ip2long - @ip2long ||= ::IPAddr.new(ip).to_i - end - - def packed_ip - @packed_ip ||= [ ip2long ].pack 'N' + def binary_ip + @binary_ip ||= ip_addr.ipv4? ? ip_addr.hton.unpack('c' * 4) : ip_addr.hton.unpack('c' * 16) end def find - tmp_offset = four_number[0] * 4 - start = IPDB.instance.index[tmp_offset..(tmp_offset + 3)].unpack("V")[0] * 8 + 1024 - - index_offset = nil - - while start < IPDB.instance.max_comp_length - if IPDB.instance.index[start..(start + 3)] >= packed_ip - index_offset = "#{IPDB.instance.index[(start + 4)..(start + 6)]}\x0".unpack("V")[0] - index_length = IPDB.instance.index[(start + 7)].unpack("C")[0] - break - end - start += 8 - end + checked = IPDB.instance.check + throw checked unless checked == :ok - return "N/A" unless index_offset + metadata = IPDB.instance.metadata + language_offset = metadata["languages"][@language] + fields_length = metadata["fields"].length + languages_length = metadata["languages"].length - result = IPDB.instance.seek(index_offset, index_length).map do |str| - str.encode("UTF-8", "UTF-8") - end + ip_string_list = IPDB.instance.resolve(binary_ip).split("\t", fields_length * languages_length) + result = ip_string_list[language_offset..language_offset + fields_length - 1] { country: result[0], province: result[1], city: result[2] } + # TODO more fields end end -end \ No newline at end of file +end diff --git a/lib/seventeen_mon/ipdb.rb b/lib/seventeen_mon/ipdb.rb index 3d385de..854a7e5 100644 --- a/lib/seventeen_mon/ipdb.rb +++ b/lib/seventeen_mon/ipdb.rb @@ -1,33 +1,134 @@ module SeventeenMon - class IPDB + require 'singleton' + require 'json' - private_class_method :new + class IPDB + include Singleton def ip_db_path - @ip_db_path ||= File.expand_path'../../data/17monipdb.dat', __FILE__ + @ip_db_path ||= File.expand_path('../data/ipip.ipdb', __dir__) + end + + def ip_db_bin + @ip_db_bin = File.binread ip_db_path + end + + # Length of the entire IPDB file. + def file_length + @file_length ||= ip_db_bin.length end - def ip_db - @ip_db ||= File.open ip_db_path, 'rb' + + # The first 4 bytes(32-bit unsigned integer) of data are used to record the length of the metadata. + META_LENGTH_OFFSET = 4 + + # Length of metadata. + def meta_length + @meta_length ||= ip_db_bin[0...META_LENGTH_OFFSET].unpack('N')[0] + end + + # Length of the entire non-data area. + def metadata_offset + @metadata_offset ||= META_LENGTH_OFFSET + meta_length end - def offset - @offset ||= ip_db.read(4).unpack("Nlen")[0] + # Metadata. + # @return [Hash] + # @example + # { + # "build" => 1535696240, + # "ip_version" => 1, + # "languages" => { "CN" => 0 }, + # "node_count" => 385083, + # "total_size" => 3117287, + # "fields" => [ "country_name", "region_name", "city_name" ] + # } + def metadata + @metadata ||= JSON.parse(ip_db_bin[META_LENGTH_OFFSET...metadata_offset]) end - def index - @index ||= ip_db.read(offset - 4) + def node_count + @node_count ||= metadata["node_count"] + end + + # Length of data. + def data_length + @data_length ||= file_length - metadata_offset + end + + def data + @data ||= ip_db_bin[metadata_offset...file_length] + end + + def check + @check ||= check_metadata + end + + def check_metadata + return "database file size error" if data_length != metadata["total_size"] + + :ok end - def max_comp_length - @max_comp_length ||= offset - 1028 + # @param [:v4, :v6] ip_version + def ip_offset(ip_version) + ip_version == :v4 ? @ip_offset_v4 ||= _ip_offset(ip_version) : @ip_offset_v6 ||= _ip_offset(ip_version) end - def self.instance - @instance ||= self.send :new + def resolve(binary_ip) + node = find_node(binary_ip) + + resoloved = node - node_count + node_count * 8 + + throw "database resolve error" if resoloved >= file_length + + size = data.byteslice(resoloved + 1, 1).unpack("c")[0] + + throw "database resolve error" if data_length < (resoloved + 2 + size) + + data.byteslice(resoloved + 2, size).encode("UTF-8", "UTF-8") + end + + private + + def find_node(binary) + bit = binary.length * 8 + node = ip_offset(bit == 32 ? :v4 : :v6) + + (0...bit).each do |i| + break if node > node_count + + node = read_node(node, 1 & ((0xFF & binary[i / 8]) >> 7 - (i % 8))) + end + + return node if node > node_count + + throw "ip not found" + end + + def _ip_offset(ip_version) + node = 0 + + return node if ip_version == :v6 + + if metadata["ip_version"] == 1 + i = 0 + while i < 96 && node < node_count + node = if i >= 80 + read_node(node, 1) + else + read_node(node, 0) + end + + i += 1 + end + end + + node end - def seek(_offset, length) - IO.read(ip_db_path, length, offset + _offset - 1024).split "\t" + def read_node(node, index) + off = node * 8 + index * 4 + data.byteslice(off, 4).unpack('N')[0] end end -end \ No newline at end of file +end diff --git a/lib/seventeen_mon/ipdb_metadata.rb b/lib/seventeen_mon/ipdb_metadata.rb new file mode 100644 index 0000000..594821c --- /dev/null +++ b/lib/seventeen_mon/ipdb_metadata.rb @@ -0,0 +1,15 @@ +module SeventeenMon + class IPDBMetadata + attr_reader :build_time, :fields, :ip_version, :languages, :node_count, :total_size, :origin_metadata + + def initialize(metadata) + @origin_metadata = metadata + @build_time = Time.at(metadata['build']) + @fields = metadata['fields'] + @ip_version = metadata['ip_version'] + @languages = metadata['languages'] + @node_count = metadata['node_count'] + @total_size = metadata['total_size'] + end + end +end diff --git a/lib/seventeen_mon/version.rb b/lib/seventeen_mon/version.rb index 4677433..5710b4d 100644 --- a/lib/seventeen_mon/version.rb +++ b/lib/seventeen_mon/version.rb @@ -1,3 +1,3 @@ module SeventeenMon - VERSION = "1.0.2" + VERSION = '2.0.0' end diff --git a/seventeen_mon.gemspec b/seventeen_mon.gemspec index 4f34254..1cddfcb 100644 --- a/seventeen_mon.gemspec +++ b/seventeen_mon.gemspec @@ -1,16 +1,15 @@ -# coding: utf-8 -lib = File.expand_path('../lib', __FILE__) +lib = File.expand_path('lib', __dir__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) require 'seventeen_mon/version' Gem::Specification.new do |spec| spec.name = "seventeen_mon" spec.version = SeventeenMon::VERSION - spec.authors = ["Jingkai He"] - spec.email = ["jaxihe@gmail.com"] - spec.summary = %q{Simply find location by IP.} - spec.description = %q{Simply find location by IP.} - spec.homepage = "https://github.com/mycolorway/SeventeenMon" + spec.authors = ["blindingdark"] + spec.email = ["blindingdark@outlook.com"] + spec.summary = "Simply find location by IP." + spec.description = "Simply find location by IP." + spec.homepage = "https://github.com/BlindingDark/SeventeenMon" spec.license = "MIT" spec.files = `git ls-files -z`.split("\x0") @@ -18,8 +17,7 @@ Gem::Specification.new do |spec| spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) spec.require_paths = ["lib"] - spec.add_development_dependency "bundler", "~> 1.5" spec.add_development_dependency "rake", "~> 10.1.1" - spec.add_development_dependency "rspec", "~> 2.14.1" + spec.add_development_dependency "rspec" end diff --git a/spec/seventeen_mon_spec.rb b/spec/seventeen_mon_spec.rb index 5d20256..a0d9237 100644 --- a/spec/seventeen_mon_spec.rb +++ b/spec/seventeen_mon_spec.rb @@ -3,10 +3,10 @@ describe SeventeenMon do describe "# IPDB loading" do it 'should be eager loaded' do - ipdb_1 = SM::IPDB.instance - ipdb_2 = SM::IPDB.instance + ipdb1 = SM::IPDB.instance + ipdb2 = SM::IPDB.instance - ipdb_1.object_id.should == ipdb_2.object_id + expect(ipdb1.object_id).to eq(ipdb2.object_id) end end @@ -14,20 +14,50 @@ before do @ip_param = "129.215.5.255" @url_param = "http://www.ruby-lang.com" + @threads = 100 end - it "can find location by ip" do + it "can find location by IP" do result = SM.find_by_ip @ip_param - result.should include(:city) - result.should include(:province) - result.should include(:country) + expect(result).to include(:city) + expect(result).to include(:province) + expect(result).to include(:country) end it "can find location by address" do result = SM.find_by_address @url_param - result.should include(:city) - result.should include(:province) - result.should include(:country) + expect(result).to include(:city) + expect(result).to include(:province) + expect(result).to include(:country) + end + + it "can find location by local IP" do + result = SM.find_by_ip '127.0.0.1' + expect(result).to include(:city) + expect(result).to include(:province) + expect(result).to include(:country) + end + + it "can run in a multi-threaded environment" do + threads = [] + @threads.times { threads << Thread.new { SM.find_by_ip(@ip_param) } } + threads.each { |t| t.join } + end + + it "can get IPDB metadata" do + expect(SM.metadata).to respond_to( + :build_time, + :fields, + :ip_version, + :languages, + :node_count, + :total_size, + :origin_metadata + ) + + require 'set' + result = Set.new(SM.metadata.origin_metadata.keys) + expect(result).to eq(Set.new(%w[build fields ip_version languages node_count total_size])) end end -end \ No newline at end of file +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 4cac63f..7bc102b 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,11 +1,10 @@ require "seventeen_mon" RSpec.configure do |config| - config.treat_symbols_as_metadata_keys_with_true_values = true config.run_all_when_everything_filtered = true config.filter_run :focus config.order = 'random' - config.color_enabled = true -end \ No newline at end of file + config.color = true +end