From ec5cc7264278285e2862f5b2f68870de29fd8086 Mon Sep 17 00:00:00 2001 From: Andrii Date: Tue, 21 May 2024 15:04:51 +0200 Subject: [PATCH] Add option to return default value if endpoint errored for json_endpoint. (#106) The current behavior is to terminate after a number of consecutive errors. With this option enabled consul-templaterb will not terminate, and return a default value instead. This is done with additional property because many users may already rely on the default behavior. Co-authored-by: a.chuzhynov --- TemplateAPI.md | 3 +- lib/consul/async/consul_template.rb | 6 +-- lib/consul/async/json_endpoint.rb | 16 +++++- spec/consul/async/json_endpoint_spec.rb | 70 +++++++++++++++++++++++-- 4 files changed, 86 insertions(+), 9 deletions(-) diff --git a/TemplateAPI.md b/TemplateAPI.md index 9af0bd1..2fc5624 100644 --- a/TemplateAPI.md +++ b/TemplateAPI.md @@ -631,11 +631,12 @@ secret('secret/foo', [force_ttl: intInSecond]) ## remote_resource -### as_json(url, default_value, [refresh_delay_secs: intInSecond]) +### as_json(url, default_value, [refresh_delay_secs: intInSecond, default_value_on_error: bool]) Fetch json data from any url. This allows to create templates with consul/vault data mixed in with data coming from other services/api. Polling interval can be controlled with `refresh_delay_secs` option. Request method (`GET`, `POST`, ...) can be controlled with `request_method` option. +To return default value on the case of error, set `default_value_on_error` to true. ```erb remote_resource.as_json('http://my-api.dev/fridge/list.json', []) diff --git a/lib/consul/async/consul_template.rb b/lib/consul/async/consul_template.rb index 82cfa3c..052fbc1 100644 --- a/lib/consul/async/consul_template.rb +++ b/lib/consul/async/consul_template.rb @@ -29,14 +29,14 @@ def initialize(endpoints_manager) @endp_manager = endpoints_manager end - def as_json(url, default_value, refresh_delay_secs: 10, **opts) + def as_json(url, default_value, refresh_delay_secs: 10, default_value_on_error: false, **opts) conf = JSONConfiguration.new(url: url, min_duration: refresh_delay_secs, retry_on_non_diff: refresh_delay_secs, **opts) endpoint_id = url + opts.hash.to_s @endp_manager.create_if_missing(url, {}, endpoint_id: endpoint_id) do if default_value.is_a?(Array) - ConsulTemplateJSONArray.new(JSONEndpoint.new(conf, url, default_value)) + ConsulTemplateJSONArray.new(JSONEndpoint.new(conf, url, default_value, default_value_on_error: default_value_on_error)) else - ConsulTemplateJSONObject.new(JSONEndpoint.new(conf, url, default_value)) + ConsulTemplateJSONObject.new(JSONEndpoint.new(conf, url, default_value, default_value_on_error: default_value_on_error)) end end end diff --git a/lib/consul/async/json_endpoint.rb b/lib/consul/async/json_endpoint.rb index bdfd9d4..16cddaa 100644 --- a/lib/consul/async/json_endpoint.rb +++ b/lib/consul/async/json_endpoint.rb @@ -93,9 +93,10 @@ def initialize(http, override_nil_response = nil) # Endpoint (aka URL) of a remote API that might be called class JSONEndpoint attr_reader :conf, :url, :queue, :stats, :last_result, :enforce_json_200, :start_time, :default_value, :query_params - def initialize(conf, url, default_value, enforce_json_200 = true, query_params = {}) + def initialize(conf, url, default_value, enforce_json_200: true, query_params: {}, default_value_on_error: false) @conf = conf.create(url) @default_value = default_value + @default_value_on_error = default_value_on_error @url = url @queue = EM::Queue.new @s_callbacks = [] @@ -174,7 +175,7 @@ def _handle_error(http) retry_in = _compute_retry_in([600, conf.retry_duration + 2**@consecutive_errors].min) ::Consul::Async::Debug.puts_error "[#{url}] - #{http.error} - Retry in #{retry_in}s #{stats.body_bytes_human}" @consecutive_errors += 1 - http_result = HttpResponse.new(http) + http_result = @default_value_on_error ? HttpResponse.new(http, @default_value.to_json) : HttpResponse.new(http) EventMachine.add_timer(retry_in) do yield queue.push(Object.new) @@ -203,6 +204,7 @@ def fetch http = connection[:conn].send(request_method, build_request) http.callback do if enforce_json_200 && !(200..299).cover?(http.response_header.status) && http.response_header['Content-Type'] != 'application/json' + handle_default_on_error(http) if @default_value_on_error _handle_error(http) do warn "[RETRY][#{url}] (#{@consecutive_errors} errors)" if (@consecutive_errors % 10) == 1 end @@ -227,6 +229,7 @@ def fetch end http.errback do + handle_default_on_error(http) if @default_value_on_error unless @stopping _handle_error(http) do if (@consecutive_errors % 10) == 1 @@ -243,6 +246,15 @@ def fetch end queue.pop(&cb) end + + def handle_default_on_error(http) + ::Consul::Async::Debug.puts_error "[#{url}] response status #{http.response_header.status}; using default value" + @consecutive_errors = 0 + json_result = JSONResult.new(@default_value.to_json, false, HttpResponse.new(http, ''), stats, 10, fake: true) + @last_result = json_result + @ready = true + @s_callbacks.each { |c| c.call(json_result) } + end end end end diff --git a/spec/consul/async/json_endpoint_spec.rb b/spec/consul/async/json_endpoint_spec.rb index 28d06ae..983272c 100644 --- a/spec/consul/async/json_endpoint_spec.rb +++ b/spec/consul/async/json_endpoint_spec.rb @@ -7,8 +7,8 @@ RSpec.describe Consul::Async do context 'default parameters' do - it 'request 200' do - mock_url = 'http://perfectly.working.url' + it 'response 200' do + mock_url = 'http://working.url' conf = Consul::Async::JSONConfiguration.new(url: mock_url) default_value = '[]' @@ -23,7 +23,7 @@ expect(json_endpoint.last_result.data).to eq(response_body.to_json) end - it 'request 500' do + it 'response 500' do mock_url = 'http://error.working.url' conf = Consul::Async::JSONConfiguration.new(url: mock_url) default_value = '' @@ -52,4 +52,68 @@ expect(json_endpoint.last_result.retry_in).to be_positive end end + + context 'when default_value_on_error on' do + let(:default_value_on_error) { true } + context 'when response 500' do + it 'return default value' do + mock_url = 'http://not.working.url' + conf = Consul::Async::JSONConfiguration.new(url: mock_url, min_duration: 10, retry_on_non_diff: 20) + default_value = '["default", "value"]' + + json_endpoint = nil + stub_request(:get, mock_url) + .to_return(body: '', status: 500) + EM.run_block do + json_endpoint = Consul::Async::JSONEndpoint.new(conf, mock_url, default_value, default_value_on_error: default_value_on_error) + end + EM.run_block do + expect(json_endpoint.ready?).to eq(true) + expect(json_endpoint.last_result.data).to eq(default_value.to_json) + expect(json_endpoint.last_result.retry_in).to be_positive + end + end + end + + context 'when address is not reachable' do + it 'return default value' do + mock_url = 'http://not.working.url' + conf = Consul::Async::JSONConfiguration.new(url: mock_url, min_duration: 10, retry_on_non_diff: 20) + default_value = '["default", "value"]' + + json_endpoint = nil + stub_request(:get, mock_url).to_timeout + EM.run_block do + json_endpoint = Consul::Async::JSONEndpoint.new(conf, mock_url, default_value, default_value_on_error: default_value_on_error) + end + EM.run_block do + expect(json_endpoint.ready?).to eq(true) + expect(json_endpoint.last_result.data).to eq(default_value.to_json) + expect(json_endpoint.last_result.retry_in).to be_positive + end + end + end + + context 'when response 200' do + it 'return value from endpoint' do + mock_url = 'http://working.url' + conf = Consul::Async::JSONConfiguration.new(url: mock_url, min_duration: 10, retry_on_non_diff: 20) + default_value = '["default", "value"]' + endpoint_body = '{"a": "b"}' + + json_endpoint = nil + stub_request(:get, mock_url) + .to_return(body: endpoint_body, status: 200) + + EM.run_block do + json_endpoint = Consul::Async::JSONEndpoint.new(conf, mock_url, default_value, default_value_on_error: default_value_on_error) + end + EM.run_block do + expect(json_endpoint.ready?).to eq(true) + expect(json_endpoint.last_result.data.to_json).to eq(endpoint_body.to_json) + expect(json_endpoint.last_result.retry_in).to be_positive + end + end + end + end end