From 2c376d95bcfdb55d78b52a8a095bf6c34fb93487 Mon Sep 17 00:00:00 2001 From: Ruud Senden <8635138+rsenden@users.noreply.github.com> Date: Mon, 18 Sep 2023 12:58:25 +0200 Subject: [PATCH] fix: Gracefully handle FoD rate limits (fixes #404) --- .../mixin/FoDProductHelperBasicMixin.java | 19 ++++++ .../helper/FoDRateLimitRetryStrategy.java | 63 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/rest/helper/FoDRateLimitRetryStrategy.java diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/output/mixin/FoDProductHelperBasicMixin.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/output/mixin/FoDProductHelperBasicMixin.java index a65709c490..22b600173a 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/output/mixin/FoDProductHelperBasicMixin.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/output/mixin/FoDProductHelperBasicMixin.java @@ -12,16 +12,21 @@ *******************************************************************************/ package com.fortify.cli.fod._common.output.mixin; +import org.apache.http.impl.client.HttpClientBuilder; + import com.fortify.cli.common.http.proxy.helper.ProxyHelper; import com.fortify.cli.common.output.product.IProductHelper; import com.fortify.cli.common.rest.unirest.config.UnirestJsonHeaderConfigurer; import com.fortify.cli.common.rest.unirest.config.UnirestUnexpectedHttpResponseConfigurer; import com.fortify.cli.common.rest.unirest.config.UnirestUrlConfigConfigurer; import com.fortify.cli.common.session.cli.mixin.AbstractSessionUnirestInstanceSupplierMixin; +import com.fortify.cli.fod._common.rest.helper.FoDRateLimitRetryStrategy; import com.fortify.cli.fod._common.session.helper.FoDSessionDescriptor; import com.fortify.cli.fod._common.session.helper.FoDSessionHelper; +import kong.unirest.Config; import kong.unirest.UnirestInstance; +import kong.unirest.apache.ApacheClient; public class FoDProductHelperBasicMixin extends AbstractSessionUnirestInstanceSupplierMixin implements IProductHelper @@ -33,6 +38,12 @@ protected final FoDSessionDescriptor getSessionDescriptor(String sessionName) { @Override protected final void configure(UnirestInstance unirest, FoDSessionDescriptor sessionDescriptor) { + // Ideally, we should be able to use unirest::config::retryAfter to handle FoD rate limits, + // but this is not possible for various reasons (see https://github.com/Kong/unirest-java/issues/491). + // As such, we use a custom ApacheClient with custom ServiceUnavailableRetryStrategy to handle + // rate-limited requests. Note that newer Unirest versions are no longer based on Apache HttpClient, + // so we'll likely need to find an alternative approach if we ever wish to upgrade to Unirest 4.x. + unirest.config().httpClient(this::createClient); UnirestUnexpectedHttpResponseConfigurer.configure(unirest); UnirestJsonHeaderConfigurer.configure(unirest); UnirestUrlConfigConfigurer.configure(unirest, sessionDescriptor.getUrlConfig()); @@ -40,4 +51,12 @@ protected final void configure(UnirestInstance unirest, FoDSessionDescriptor ses final String authHeader = String.format("Bearer %s", sessionDescriptor.getActiveBearerToken()); unirest.config().setDefaultHeader("Authorization", authHeader); } + + private ApacheClient createClient(Config config) { + return new ApacheClient(config, this::configureClient); + } + + private void configureClient(HttpClientBuilder cb) { + cb.setServiceUnavailableRetryStrategy(new FoDRateLimitRetryStrategy()); + } } \ No newline at end of file diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/rest/helper/FoDRateLimitRetryStrategy.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/rest/helper/FoDRateLimitRetryStrategy.java new file mode 100644 index 0000000000..2e4f5bf8c1 --- /dev/null +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/rest/helper/FoDRateLimitRetryStrategy.java @@ -0,0 +1,63 @@ +/******************************************************************************* + * (c) Copyright 2020 Micro Focus or one of its affiliates, a Micro Focus company + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, + * sublicense, and/or sell copies of the Software, and to permit persons to + * whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ +package com.fortify.cli.fod._common.rest.helper; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.HttpResponse; +import org.apache.http.client.ServiceUnavailableRetryStrategy; +import org.apache.http.protocol.HttpContext; + +/** + * This class implements an Apache HttpClient 4.x {@link ServiceUnavailableRetryStrategy} + * that will retry a request if the server responds with an HTTP 429 (TOO_MANY_REQUESTS) + * response. + */ +public final class FoDRateLimitRetryStrategy implements ServiceUnavailableRetryStrategy { + private static final Log LOG = LogFactory.getLog(FoDRateLimitRetryStrategy.class); + private final String HEADER_NAME = "X-Rate-Limit-Reset"; + private int maxRetries = 2; + private final ThreadLocal interval = new ThreadLocal(); + + public FoDRateLimitRetryStrategy maxRetries(int maxRetries) { + this.maxRetries = maxRetries; + return this; + } + + public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) { + if ( executionCount < maxRetries+1 && response.getStatusLine().getStatusCode()==429 ) { + int retrySeconds = Integer.parseInt(response.getFirstHeader(HEADER_NAME).getValue()); + LOG.debug("Rate-limited request will be retried after "+retrySeconds+" seconds"); + interval.set((long)retrySeconds*1000); + return true; + } + return false; + } + + public long getRetryInterval() { + Long result = interval.get(); + return result==null ? -1 : result; + } +}