diff --git a/packages/ruby/Gemfile b/packages/ruby/Gemfile index 6b9b01a942..64fe1eae1d 100644 --- a/packages/ruby/Gemfile +++ b/packages/ruby/Gemfile @@ -11,3 +11,5 @@ gem "rake", "~> 12.0" gem "rspec", "~> 3.0" gem "standard" gem "webmock" +gem "os" +gem "uuid" diff --git a/packages/ruby/Gemfile.lock b/packages/ruby/Gemfile.lock index 164cfab6d6..d180fcdea5 100644 --- a/packages/ruby/Gemfile.lock +++ b/packages/ruby/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - readme-metrics (1.1.0) + readme-metrics (1.1.1) httparty (~> 0.18) GEM @@ -14,15 +14,18 @@ GEM safe_yaml (~> 1.0.0) diff-lcs (1.4.4) hashdiff (1.0.1) - httparty (0.18.1) + httparty (0.20.0) mime-types (~> 3.0) multi_xml (>= 0.5.2) json-schema (2.8.1) addressable (>= 2.4) - mime-types (3.3.1) + macaddr (1.7.2) + systemu (~> 2.6.5) + mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0225) + mime-types-data (3.2021.1115) multi_xml (0.6.0) + os (1.1.4) parallel (1.19.2) parser (2.7.1.4) ast (~> 2.4.1) @@ -65,7 +68,10 @@ GEM standard (0.4.7) rubocop (~> 0.85.0) rubocop-performance (~> 1.6.0) + systemu (2.6.5) unicode-display_width (1.7.0) + uuid (2.3.9) + macaddr (~> 1.0) webmock (3.8.3) addressable (>= 2.3.6) crack (>= 0.3.2) @@ -76,11 +82,13 @@ PLATFORMS DEPENDENCIES json-schema + os rack-test rake (~> 12.0) readme-metrics! rspec (~> 3.0) standard + uuid webmock BUNDLED WITH diff --git a/packages/ruby/README.md b/packages/ruby/README.md index 7ad94bd01d..b6b1345091 100644 --- a/packages/ruby/README.md +++ b/packages/ruby/README.md @@ -25,25 +25,25 @@ from the environment, or you may hardcode them. If you're using Warden-based authentication like Devise, you may fetch the current_user for a given request from the environment. -### Batching requests - -By default, the middleware will batch requests to the ReadMe API in groups of 10. -For every 10 requests made to your application, the middleware will make a -single request to ReadMe. If you wish to override this, provide a -`buffer_length` option when configuring the middleware. - -### Sensitive Data - -If you have sensitive data you'd like to prevent from being sent to the Metrics -API via headers, query params or payload bodies, you can specify a list of keys -to filter via the `reject_params` option. Key-value pairs matching these keys -will not be included in the request to the Metrics API. - -You are also able to specify a set of `allow_only` which should only be sent through. -Any header or body values not matching these keys will be filtered out and not -send to the API. - -You may only specify either `reject_params` or `allow_only` keys, not both. +### SDK Options + +Option | Type | Description +-----------------|------------------|--------- +`reject_params` | Array of strings | If you have sensitive data you'd like to prevent from being sent to the Metrics API via headers, query params or payload bodies, you can specify a list of keys +to filter via the `reject_params` option. NOTE: cannot be used in conjunction with `allow_only`. You may only specify either `reject_params` or `allow_only` keys, not both. +`allow_only` | Array of strings | The inverse of `reject_params`. If included all parameters but those in this list will be redacted. NOTE: cannot be used in conjunction with `reject_params`. You may only specify either `reject_params` or `allow_only` keys, not both. +`development` | bool | Defaults to `false`. When `true`, the log will be marked as a development log. This is great for separating staging or test data from data coming from customers. +`buffer_length` | number | Defaults to `1`. This value should be a number representing the amount of requests to group up before sending them over the network. Increasing this value may increase performance by batching, but will also delay the time until logs show up in the dashboard given the buffer size needs to be reached in order for the logs to be sent. + +### Payload Data + +Option | Required? | Type | Description +--------------------|-----------|------------------|---------- +`api_key` | yes | string | API Key used to make the request. Note that this is different from the `readmeAPIKey` described above in the options data. This should be a value from your API that is unique to each of your users. +`label` | no | string | This will be the user's display name in the API Metrics Dashboard, since it's much easier to remember a name than an API key. +`email` | no | string | Email of the user that is making the call. +`log_id` | no | string | A UUIDv4 identifier. If not provided this will be automatically generated for you. Providing your own `log_id` is useful if you want to know the URL of the log in advance, i.e. `{your_base_url}/logs/{your_log_id}`. +`ignore` | no | bool | A flag that when set to `true` will suppress sending the log. ### Rails @@ -51,29 +51,27 @@ You may only specify either `reject_params` or `allow_only` keys, not both. # config/environments/development.rb or config/environments/production.rb require "readme/metrics" -options = { +sdk_options = { api_key: "<>", development: false, reject_params: ["not_included", "dont_send"], buffer_length: 5, } -config.middleware.use Readme::Metrics, options do |env| +config.middleware.use Readme::Metrics, sdk_options do |env| current_user = env['warden'].authenticate - if current_user.present? - { - api_key: current_user.api_key, # Not the same as the ReadMe API Key - label: current_user.name, - email: current_user.email - } - else - { - api_key: "guest", - label: "Guest User", - email: "guest@example.com" - } - end + payload_data = current_user.present? ? { + api_key: current_user.api_key, # Not the same as the ReadMe API Key + label: current_user.name, + email: current_user.email + } : { + api_key: "guest", + label: "Guest User", + email: "guest@example.com" + } + + payload_data end ``` @@ -81,17 +79,18 @@ end ```ruby # config.ru -options = { +sdk_options = { api_key: "<>", development: false, reject_params: ["not_included", "dont_send"] } -use Readme::Metrics, options do |env| +use Readme::Metrics, sdk_options do |env| { api_key: "owlbert_api_key" label: "Owlbert", - email: "owlbert@example.com" + email: "owlbert@example.com", + log_id: SecureRandom.uuid } end @@ -104,6 +103,10 @@ run YourApp.new - [Rack](https://github.com/readmeio/metrics-sdk-racks-sample) - [Sinatra](https://github.com/readmeio/metrics-sdk-sinatra-example) +### Contributing + +Ensure you are running the version of ruby specified in the `Gemfile.lock`; use `rvm` to easy manage ruby versions. Run `bundle` to install dependencies, `rake` or `rspec` to ensure tests pass, and `bundle exec standardrb` to lint the code. + ## License [View our license here](https://github.com/readmeio/metrics-sdks/tree/main/packages/ruby/LICENSE) diff --git a/packages/ruby/lib/readme/har/request_serializer.rb b/packages/ruby/lib/readme/har/request_serializer.rb index efdca9aa77..314c613ccc 100644 --- a/packages/ruby/lib/readme/har/request_serializer.rb +++ b/packages/ruby/lib/readme/har/request_serializer.rb @@ -1,10 +1,11 @@ +require "cgi" require "readme/har/collection" require "readme/filter" module Readme module Har class RequestSerializer - def initialize(request, filter = Filter::None.new) + def initialize(request, filter = Readme::Filter::None.new) @request = request @filter = filter end @@ -13,7 +14,7 @@ def as_json { method: @request.request_method, queryString: Har::Collection.new(@filter, @request.query_params).to_a, - url: @request.url, + url: url, httpVersion: @request.http_version, headers: Har::Collection.new(@filter, @request.headers).to_a, cookies: Har::Collection.new(@filter, @request.cookies).to_a, @@ -25,6 +26,16 @@ def as_json private + def url + url = URI(@request.url) + headers = @request.headers + forward_proto = headers["X-Forwarded-Proto"] + forward_host = headers["X-Forwarded-Host"] + url.host = forward_host if forward_host.is_a?(String) + url.scheme = forward_proto if forward_proto.is_a?(String) + url.to_s + end + def postData if @request.content_type.nil? nil @@ -48,18 +59,34 @@ def form_encoded_body def request_body if @filter.pass_through? pass_through_body - else - # Only JSON allowed for non-pass-through situations. It will raise - # if the body can't be parsed as JSON, aborting the request. + elsif is_form_urlencoded? + form_urlencoded_body + elsif is_json? json_body + else + @request.body end end + def is_json? + ["application/json", "application/x-json", "text/json", "text/x-json"] + .include?(@request.content_type) || @request.content_type.include?("+json") + end + + def is_form_urlencoded? + @request.content_type == "application/x-www-form-urlencoded" + end + def json_body parsed_body = JSON.parse(@request.body) Har::Collection.new(@filter, parsed_body).to_h.to_json end + def form_urlencoded_body + parsed_body = CGI.parse(@request.body).transform_values(&:first) + Har::Collection.new(@filter, parsed_body).to_h.to_json + end + def pass_through_body @request.body end diff --git a/packages/ruby/lib/readme/har/serializer.rb b/packages/ruby/lib/readme/har/serializer.rb index 09af5e4d30..73ab36f15f 100644 --- a/packages/ruby/lib/readme/har/serializer.rb +++ b/packages/ruby/lib/readme/har/serializer.rb @@ -32,7 +32,8 @@ def to_json def creator { name: Readme::Metrics::SDK_NAME, - version: Readme::Metrics::VERSION + version: Readme::Metrics::VERSION, + comment: "#{Readme::Metrics::PLATFORM}/#{RUBY_VERSION}" } end diff --git a/packages/ruby/lib/readme/metrics.rb b/packages/ruby/lib/readme/metrics.rb index 2d237e1bea..8b6ce81c3e 100644 --- a/packages/ruby/lib/readme/metrics.rb +++ b/packages/ruby/lib/readme/metrics.rb @@ -8,14 +8,26 @@ require "readme/http_response" require "httparty" require "logger" +require "os" module Readme class Metrics + def self.platform + if OS.windows? + "windows" + elsif OS.mac? + "mac" + elsif OS.linux? + "linux" + else + "unknown" + end + end + SDK_NAME = "Readme.io Ruby SDK" - DEFAULT_BUFFER_LENGTH = 10 + PLATFORM = platform + DEFAULT_BUFFER_LENGTH = 1 ENDPOINT = "https://metrics.readme.io/v1/request" - USER_INFO_KEYS = [:api_key, :label, :email] - USER_INFO_KEYS_DEPRECATED = [:id, :label, :email] def self.logger @@logger @@ -74,7 +86,7 @@ def process_response(response:, env:, start_time:, end_time:) Readme::Metrics.logger.warn "Request or response body MIME type isn't supported for filtering. Omitting request from ReadMe API logging" else payload = Payload.new(har, user_info, development: @development) - @@request_queue.push(payload.to_json) + @@request_queue.push(payload.to_json) unless payload.ignore end end @@ -134,11 +146,9 @@ def is_a_boolean?(arg) end def user_info_valid?(user_info) - sorted_user_info_keys = user_info.keys.sort !user_info.nil? && !user_info.values.any?(&:nil?) && - (sorted_user_info_keys === USER_INFO_KEYS.sort || - sorted_user_info_keys === USER_INFO_KEYS_DEPRECATED.sort) + user_info.has_key?(:api_key) || user_info.has_key?(:id) end end end diff --git a/packages/ruby/lib/readme/payload.rb b/packages/ruby/lib/readme/payload.rb index b185e371d1..a933f612fa 100644 --- a/packages/ruby/lib/readme/payload.rb +++ b/packages/ruby/lib/readme/payload.rb @@ -1,15 +1,22 @@ +require "uuid" + module Readme class Payload - def initialize(har, user_info, development:) + attr_reader :ignore + + def initialize(har, info, development:) @har = har - # swap api_key for id - user_info[:id] = user_info.delete :api_key unless user_info[:api_key].nil? - @user_info = user_info + @user_info = info.slice(:id, :label, :email) + @user_info[:id] = info[:api_key] unless info[:api_key].nil? # swap api_key for id if api_key is present + @log_id = info[:log_id] + @ignore = info[:ignore] @development = development + @uuid = UUID.new end def to_json { + logId: UUID.validate(@log_id) ? @log_id : @uuid.generate, group: @user_info, clientIPAddress: "1.1.1.1", development: @development, diff --git a/packages/ruby/spec/readme/har/request_serializer_spec.rb b/packages/ruby/spec/readme/har/request_serializer_spec.rb index 846814118a..04578877f7 100644 --- a/packages/ruby/spec/readme/har/request_serializer_spec.rb +++ b/packages/ruby/spec/readme/har/request_serializer_spec.rb @@ -116,6 +116,50 @@ expect { serializer.as_json }.to raise_error(JSON::ParserError) end end + + it "respects forwarded headers" do + http_request = build_http_request( + url: "http://example.com/api/foo/bar?id=1&name=joel", + content_type: "application/json", + headers: { + "X-Forwarded-Proto" => "https", + "X-Forwarded-Host" => "www.example.edu" + }, + body: {key1: "key1", key2: "key2"}.to_json + ) + + request = Readme::Har::RequestSerializer.new(http_request) + json = request.as_json + + expect(json[:url]).to eq "https://www.example.edu/api/foo/bar?id=1&name=joel" + end + + it "parses multiple json content types" do + http_request = build_http_request( + content_type: "application/x-json", + body: {key1: "value1", key2: "value2"}.to_json + ) + + request = Readme::Har::RequestSerializer.new(http_request, Readme::Filter::RejectParams.new([])) + json = request.as_json + + expect(json.dig(:postData, :text)).to eq http_request.body + end + + it "parses form-urlencoded content type" do + http_request = build_http_request( + content_type: "application/x-www-form-urlencoded", + body: "key1=value1&key2=value2", + query_params: {}, + url: "https://example.com/" + ) + + request = Readme::Har::RequestSerializer.new(http_request, Readme::Filter::RejectParams.new([])) + json = request.as_json + expected = {key1: "value1", key2: "value2"}.to_json + + expect(json.dig(:postData, :text)).to eq expected + end end # if overriding `url` to have query parameters make sure to also override diff --git a/packages/ruby/spec/readme/har/response_serializer_spec.rb b/packages/ruby/spec/readme/har/response_serializer_spec.rb index 62ef025ab7..5a631e7641 100644 --- a/packages/ruby/spec/readme/har/response_serializer_spec.rb +++ b/packages/ruby/spec/readme/har/response_serializer_spec.rb @@ -2,6 +2,8 @@ require "readme/filter" RSpec.describe Readme::Har::ResponseSerializer do + Filter = Readme::Filter + describe "#as_json" do it "creates a structure that is valid according the schema" do request = build_request diff --git a/packages/ruby/spec/readme/har/serializer_spec.rb b/packages/ruby/spec/readme/har/serializer_spec.rb index 5540252db6..bc05cdee4e 100644 --- a/packages/ruby/spec/readme/har/serializer_spec.rb +++ b/packages/ruby/spec/readme/har/serializer_spec.rb @@ -24,7 +24,7 @@ double(:rack_response), start_time, end_time, - Filter::None.new + Readme::Filter::None.new ) json = JSON.parse(har.to_json) @@ -33,6 +33,7 @@ expect(json.dig("log", "version")).to eq Readme::Har::Serializer::HAR_VERSION expect(json.dig("log", "creator", "name")).to eq Readme::Metrics::SDK_NAME expect(json.dig("log", "creator", "version")).to eq Readme::Metrics::VERSION + expect(json.dig("log", "creator", "comment")).to eq "#{Readme::Metrics::PLATFORM}/#{RUBY_VERSION}" expect(json.dig("log", "entries").length).to eq 1 expect(json.dig("log", "entries", 0, "cache")).to be_empty expect(json.dig("log", "entries", 0, "timings", "send")).to eq 0 diff --git a/packages/ruby/spec/readme/metrics_spec.rb b/packages/ruby/spec/readme/metrics_spec.rb index 0d0ea6aa0f..0e35511266 100644 --- a/packages/ruby/spec/readme/metrics_spec.rb +++ b/packages/ruby/spec/readme/metrics_spec.rb @@ -1,10 +1,15 @@ require "readme/metrics" require "rack/test" require "webmock/rspec" +require "uuid" RSpec.describe Readme::Metrics do include Rack::Test::Methods + before do + @uuid = UUID.new + end + before :each do WebMock.reset_executed_requests! end @@ -429,6 +434,16 @@ def app .with { |request| validate_json("readmeMetrics", request.body) } }.to raise_error end + + it "can ignore sending logs" do + def app + json_app_with_middleware({}, {ignore: true}) + end + + post "/api/foo" + + expect(WebMock).not_to have_requested(:post, Readme::Metrics::ENDPOINT) + end end def json_app_with_middleware(option_overrides = {}, group_overrides = {}) @@ -449,7 +464,9 @@ def app_with_middleware(app, option_overrides = {}, group_overrides = {}) group = { id: env["CURRENT_USER"].id, label: env["CURRENT_USER"].name, - email: env["CURRENT_USER"].email + email: env["CURRENT_USER"].email, + log_id: @uuid.generate, + ignore: false }.merge(group_overrides) group.delete :id unless group[:api_key].nil? group diff --git a/packages/ruby/spec/readme/payload_spec.rb b/packages/ruby/spec/readme/payload_spec.rb index 1c7414bd72..25eb39ad50 100644 --- a/packages/ruby/spec/readme/payload_spec.rb +++ b/packages/ruby/spec/readme/payload_spec.rb @@ -1,9 +1,11 @@ require "readme/payload" +require "uuid" RSpec.describe Readme::Payload do before :each do har_json = File.read(File.expand_path("../../fixtures/har.json", __FILE__)) @har = double("har", to_json: har_json) + @uuid = UUID.new end it "returns JSON matching the payload schema" do @@ -25,4 +27,27 @@ expect(result.to_json).to match_json_schema("payload") end + + it "accepts a custom log uuid" do + uuid = @uuid.generate + result = Readme::Payload.new( + @har, + {api_key: "1", label: "Owlbert", email: "owlbert@example.com", log_id: uuid}, + development: true + ) + + expect(JSON.parse(result.to_json)).to include("logId" => uuid) + expect(result.to_json).to match_json_schema("payload") + end + + it "rejects an invalid log uuid" do + result = Readme::Payload.new( + @har, + {api_key: "1", label: "Owlbert", email: "owlbert@example.com", log_id: "invalid"}, + development: true + ) + + expect(JSON.parse(result.to_json)).to_not include("logId" => "invalid") + expect(result.to_json).to match_json_schema("payload") + end end diff --git a/packages/ruby/spec/schema/payload.json b/packages/ruby/spec/schema/payload.json index 306e484206..101a822643 100644 --- a/packages/ruby/spec/schema/payload.json +++ b/packages/ruby/spec/schema/payload.json @@ -2,6 +2,9 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "properties": { + "logId": { + "type": "string" + }, "group": { "$ref": "readmeGroup.json#" },