diff --git a/README.md b/README.md index 9d97558..b401261 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ end - [Applebot](https://support.apple.com/en-us/119829) - [Baidu spider](http://help.baidu.com/question?prod_en=master&class=498&id=1000973) - [Bingbot](https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/) +- [DataForSEO](https://dataforseo.com/dataforseo-bot) - [DuckDuckGo bot](https://duckduckgo.com/duckduckbot) - [Google crawlers](https://support.google.com/webmasters/answer/1061943) - [IAS](https://integralads.com/ias-privacy-data-management/policies/site-indexing-policy/) diff --git a/lib/legitbot.rb b/lib/legitbot.rb index 957fb02..b9fc249 100644 --- a/lib/legitbot.rb +++ b/lib/legitbot.rb @@ -8,6 +8,7 @@ require_relative 'legitbot/apple' require_relative 'legitbot/baidu' require_relative 'legitbot/bing' +require_relative 'legitbot/dataforseo' require_relative 'legitbot/duckduckgo' require_relative 'legitbot/facebook' require_relative 'legitbot/google' diff --git a/lib/legitbot/dataforseo.rb b/lib/legitbot/dataforseo.rb new file mode 100644 index 0000000..3ce2da2 --- /dev/null +++ b/lib/legitbot/dataforseo.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Legitbot # :nodoc: + # https://dataforseo.com/dataforseo-bot + class DataForSEO < BotMatch + domains 'dataforseo.com.' + end + + rule Legitbot::DataForSEO, %w[DataForSeoBot] +end diff --git a/test/dataforseo_test.rb b/test/dataforseo_test.rb new file mode 100644 index 0000000..cd105ca --- /dev/null +++ b/test/dataforseo_test.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require_relative 'test_helper' + +class DataForSEOTest < Minitest::Test + include Minitest::Hooks + include DnsServerMock + + def test_malicious_ip + ip = '149.210.164.47' + match = Legitbot::DataForSEO.new ip + + refute_predicate match, :valid? + end + + def test_valid_ip + ip = '136.243.228.176' + match = Legitbot::DataForSEO.new ip + + assert_predicate match, :valid? + end + + def test_malicious_ua + bot = Legitbot.bot( + 'Mozilla/5.0 (compatible; DataForSeoBot; +https://dataforseo.com/dataforseo-bot)', + '149.210.164.47' + ) + + assert bot + refute_predicate bot, :valid? + end + + def test_valid_ua + bot = Legitbot.bot( + 'Mozilla/5.0 (compatible; DataForSeoBot; +https://dataforseo.com/dataforseo-bot)', + '136.243.228.176' + ) + + assert bot + assert_predicate bot, :valid? + end + + def test_valid_name + bot = Legitbot.bot( + 'Mozilla/5.0 (compatible; DataForSeoBot; +https://dataforseo.com/dataforseo-bot)', + '136.243.228.176' + ) + + assert_equal :dataforseo, bot.detected_as + end + + def test_fake_name + bot = Legitbot.bot( + 'Mozilla/5.0 (compatible; DataForSeoBot; +https://dataforseo.com/dataforseo-bot)', + '81.1.172.108' + ) + + assert_equal :dataforseo, bot.detected_as + end +end diff --git a/test/lib/dns_server_mock.rb b/test/lib/dns_server_mock.rb index fd0dc04..4d99eba 100644 --- a/test/lib/dns_server_mock.rb +++ b/test/lib/dns_server_mock.rb @@ -44,6 +44,14 @@ ptr: %w[17-58-98-60.applebot.apple.com] }, + # DataForSEO + 'crawling-gateway-136-243-228-176.dataforseo.com' => { + a: %w[136.243.228.176] + }, + '136.243.228.176' => { + ptr: %w[crawling-gateway-136-243-228-176.dataforseo.com] + }, + # Google 'crawl-66-249-64-141.googlebot.com' => { a: %w[66.249.64.141]