From 78d397345b8801f4cd9c0b347ed6f62ec2f5a932 Mon Sep 17 00:00:00 2001 From: Marcos Barbero Date: Sun, 25 Oct 2020 10:43:46 +0100 Subject: [PATCH] Add request origin deny list (#390) * Add origin deny list feature * Add integration tests * Update greetings.yml --- .github/workflows/greetings.yml | 2 +- README.adoc | 9 +++ .../properties/RateLimitProperties.java | 71 +++++++++++++++---- .../repository/AbstractRateLimiter.java | 2 +- .../filters/AbstractRateLimitFilter.java | 31 ++++++-- .../ratelimit/filters/RateLimitPreFilter.java | 2 + .../support/RateLimitExceededException.java | 4 ++ .../filters/pre/RateLimitPreFilterTest.java | 24 +++---- .../src/main/resources/application.yml | 1 - .../tests/it/SpringDataApplicationTestIT.java | 29 +++----- .../tests/it/SpringDataDenyOriginTestIT.java | 30 ++++++++ 11 files changed, 152 insertions(+), 53 deletions(-) create mode 100644 spring-cloud-zuul-ratelimit-tests/springdata/src/test/java/com/marcosbarbero/tests/it/SpringDataDenyOriginTestIT.java diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml index bae1c29f..9f5bd985 100644 --- a/.github/workflows/greetings.yml +++ b/.github/workflows/greetings.yml @@ -1,6 +1,6 @@ name: Greetings -on: [pull_request, issues] +on: [issues] jobs: greeting: diff --git a/README.adoc b/README.adoc index 9a07b989..e5524e12 100644 --- a/README.adoc +++ b/README.adoc @@ -174,6 +174,11 @@ zuul: repository: REDIS behind-proxy: true add-response-headers: true + default-deny-list: + response-status-code: 404 #default value is 403 (FORBIDDEN) + origins: + - 200.187.10.25 + - somedomain.com default-policy-list: #optional - will apply unless specific policy exists - limit: 10 #optional - request number limit per refresh interval window quota: 1000 #optional - request time limit per refresh interval window (in seconds) @@ -212,6 +217,10 @@ zuul.ratelimit.repository=REDIS zuul.ratelimit.behind-proxy=true zuul.ratelimit.add-response-headers=true +zuul.ratelimit.default-deny-list.response-status-code=404 +zuul.ratelimit.default-deny-list.origins[0]=200.187.10.25 +zuul.ratelimit.default-deny-list.origins[1]=somedomain.com + zuul.ratelimit.default-policy-list[0].limit=10 zuul.ratelimit.default-policy-list[0].quota=1000 zuul.ratelimit.default-policy-list[0].refresh-interval=60 diff --git a/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/config/properties/RateLimitProperties.java b/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/config/properties/RateLimitProperties.java index bf2d3225..ea7041c6 100644 --- a/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/config/properties/RateLimitProperties.java +++ b/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/config/properties/RateLimitProperties.java @@ -16,22 +16,10 @@ package com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.config.properties; -import static com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.config.properties.ResponseHeadersVerbosity.NONE; -import static com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.config.properties.ResponseHeadersVerbosity.VERBOSE; -import static org.springframework.cloud.netflix.zuul.filters.support.FilterConstants.FORM_BODY_WRAPPER_FILTER_ORDER; -import static org.springframework.cloud.netflix.zuul.filters.support.FilterConstants.SEND_RESPONSE_FILTER_ORDER; - import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.config.RateLimitUtils; import com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.config.properties.validators.Policies; -import java.time.Duration; -import java.time.temporal.ChronoUnit; -import java.util.List; -import java.util.Map; -import javax.servlet.http.HttpServletRequest; -import javax.validation.Valid; -import javax.validation.constraints.NotNull; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -40,8 +28,22 @@ import org.springframework.boot.convert.DurationUnit; import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.cloud.netflix.zuul.filters.Route; +import org.springframework.http.HttpStatus; import org.springframework.validation.annotation.Validated; +import javax.servlet.http.HttpServletRequest; +import javax.validation.Valid; +import javax.validation.constraints.NotNull; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Map; + +import static com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.config.properties.ResponseHeadersVerbosity.NONE; +import static com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.config.properties.ResponseHeadersVerbosity.VERBOSE; +import static org.springframework.cloud.netflix.zuul.filters.support.FilterConstants.FORM_BODY_WRAPPER_FILTER_ORDER; +import static org.springframework.cloud.netflix.zuul.filters.support.FilterConstants.SEND_RESPONSE_FILTER_ORDER; + /** * @author Marcos Barbero * @author Liel Chayoun @@ -83,6 +85,9 @@ public class RateLimitProperties { private int preFilterOrder = FORM_BODY_WRAPPER_FILTER_ORDER; + @NestedConfigurationProperty + private DenyList defaultDenyList = new DenyList(); + public List getPolicies(String key) { return policyList.getOrDefault(key, defaultPolicyList); } @@ -135,11 +140,11 @@ public void setAddResponseHeaders(boolean addResponseHeaders) { } public ResponseHeadersVerbosity getResponseHeaders() { - return this.responseHeaders; + return this.responseHeaders; } public void setResponseHeaders(ResponseHeadersVerbosity responseHeaders) { - this.responseHeaders = responseHeaders; + this.responseHeaders = responseHeaders; } public String getKeyPrefix() { @@ -174,6 +179,14 @@ public void setPreFilterOrder(int preFilterOrder) { this.preFilterOrder = preFilterOrder; } + public DenyList getDefaultDenyList() { + return defaultDenyList; + } + + public void setDefaultDenyList(DenyList defaultDenyList) { + this.defaultDenyList = defaultDenyList; + } + public static class Policy { /** * Refresh interval window (in seconds). @@ -279,4 +292,34 @@ public void setMatcher(String matcher) { } } } + + public static class DenyList { + + /** + * List of origins that will have the request denied. + */ + @NotNull + private List origins = Lists.newArrayList(); + + /** + * Status code returned when a blocked origin tries to reach the server. + */ + private int responseStatusCode = HttpStatus.FORBIDDEN.value(); + + public List getOrigins() { + return origins; + } + + public void setOrigins(List origins) { + this.origins = origins; + } + + public int getResponseStatusCode() { + return responseStatusCode; + } + + public void setResponseStatusCode(int responseStatusCode) { + this.responseStatusCode = responseStatusCode; + } + } } diff --git a/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/config/repository/AbstractRateLimiter.java b/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/config/repository/AbstractRateLimiter.java index 35a4ec85..f6b59a1b 100644 --- a/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/config/repository/AbstractRateLimiter.java +++ b/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/config/repository/AbstractRateLimiter.java @@ -66,7 +66,7 @@ private Rate create(final Policy policy, final String key) { Long limit = policy.getLimit(); Long quota = policy.getQuota() != null ? policy.getQuota().toMillis() : null; - Long refreshInterval = policy.getRefreshInterval().toMillis(); + long refreshInterval = policy.getRefreshInterval().toMillis(); Date expiration = new Date(System.currentTimeMillis() + refreshInterval); return new Rate(key, limit, quota, refreshInterval, expiration); diff --git a/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/filters/AbstractRateLimitFilter.java b/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/filters/AbstractRateLimitFilter.java index 9fa9770a..1782685d 100644 --- a/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/filters/AbstractRateLimitFilter.java +++ b/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/filters/AbstractRateLimitFilter.java @@ -20,19 +20,20 @@ import com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.config.properties.RateLimitProperties; import com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.config.properties.RateLimitProperties.Policy; import com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.config.properties.RateLimitProperties.Policy.MatchType; +import com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.config.properties.RateLimitType; +import com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.support.RateLimitExceededException; import com.netflix.zuul.ZuulFilter; import com.netflix.zuul.context.RequestContext; import org.springframework.cloud.netflix.zuul.filters.Route; import org.springframework.cloud.netflix.zuul.filters.RouteLocator; +import org.springframework.http.HttpStatus; import org.springframework.web.util.UrlPathHelper; import javax.servlet.http.HttpServletRequest; import java.util.List; import java.util.stream.Collectors; -import static com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.support.RateLimitConstants.ALREADY_LIMITED; -import static com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.support.RateLimitConstants.CURRENT_REQUEST_POLICY; -import static com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.support.RateLimitConstants.CURRENT_REQUEST_ROUTE; +import static com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.support.RateLimitConstants.*; /** @@ -57,8 +58,22 @@ abstract class AbstractRateLimitFilter extends ZuulFilter { @Override public boolean shouldFilter() { - HttpServletRequest request = RequestContext.getCurrentContext().getRequest(); - return properties.isEnabled() && !policy(route(request), request).isEmpty(); + RequestContext ctx = RequestContext.getCurrentContext(); + HttpServletRequest request = ctx.getRequest(); + + if (!properties.isEnabled()) { + return false; + } + + if (originIsOnDefaultDenyList(request)) { + int responseStatusCode = properties.getDefaultDenyList().getResponseStatusCode(); + ctx.setResponseStatusCode(responseStatusCode); + ctx.setSendZuulResponse(false); + + throw new RateLimitExceededException(HttpStatus.valueOf(responseStatusCode)); + } + + return !policy(route(request), request).isEmpty(); } Route route(HttpServletRequest request) { @@ -95,6 +110,12 @@ protected List policy(Route route, HttpServletRequest request) { return policies; } + private boolean originIsOnDefaultDenyList(HttpServletRequest request) { + RateLimitProperties.DenyList denyList = properties.getDefaultDenyList(); + return denyList.getOrigins().stream() + .anyMatch(origin -> RateLimitType.ORIGIN.apply(request, null, rateLimitUtils, origin)); + } + private void addObjectToCurrentRequestContext(String key, Object object) { if (object != null) { RequestContext.getCurrentContext().put(key, object); diff --git a/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/filters/RateLimitPreFilter.java b/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/filters/RateLimitPreFilter.java index 68f9ae99..8615be23 100644 --- a/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/filters/RateLimitPreFilter.java +++ b/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/filters/RateLimitPreFilter.java @@ -37,10 +37,12 @@ import com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.support.RateLimitExceededEvent; import com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.support.RateLimitExceededException; import com.netflix.zuul.context.RequestContext; + import java.time.Duration; import java.util.Map; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; + import org.springframework.cloud.netflix.zuul.filters.Route; import org.springframework.cloud.netflix.zuul.filters.RouteLocator; import org.springframework.context.ApplicationEventPublisher; diff --git a/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/support/RateLimitExceededException.java b/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/support/RateLimitExceededException.java index 837f9289..0bf9f998 100644 --- a/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/support/RateLimitExceededException.java +++ b/spring-cloud-zuul-ratelimit-core/src/main/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/support/RateLimitExceededException.java @@ -12,4 +12,8 @@ public class RateLimitExceededException extends ZuulRuntimeException { public RateLimitExceededException() { super(new ZuulException(HttpStatus.TOO_MANY_REQUESTS.toString(), HttpStatus.TOO_MANY_REQUESTS.value(), null)); } + + public RateLimitExceededException(HttpStatus httpStatus) { + super(new ZuulException(httpStatus.toString(), httpStatus.value(), null)); + } } diff --git a/spring-cloud-zuul-ratelimit-core/src/test/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/filters/pre/RateLimitPreFilterTest.java b/spring-cloud-zuul-ratelimit-core/src/test/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/filters/pre/RateLimitPreFilterTest.java index e1166456..6619905c 100644 --- a/spring-cloud-zuul-ratelimit-core/src/test/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/filters/pre/RateLimitPreFilterTest.java +++ b/spring-cloud-zuul-ratelimit-core/src/test/java/com/marcosbarbero/cloud/autoconfigure/zuul/ratelimit/filters/pre/RateLimitPreFilterTest.java @@ -1,14 +1,5 @@ package com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.filters.pre; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - import com.google.common.collect.Lists; import com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.config.Rate; import com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.config.RateLimitKeyGenerator; @@ -25,11 +16,7 @@ import com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.support.RateLimitExceededException; import com.netflix.zuul.context.RequestContext; import com.netflix.zuul.monitoring.CounterFactory; -import java.util.Collections; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.mockito.Captor; @@ -43,6 +30,17 @@ import org.springframework.web.context.request.RequestContextHolder; import org.springframework.web.util.UrlPathHelper; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + public class RateLimitPreFilterTest { private RateLimitPreFilter target; diff --git a/spring-cloud-zuul-ratelimit-tests/springdata/src/main/resources/application.yml b/spring-cloud-zuul-ratelimit-tests/springdata/src/main/resources/application.yml index 2fc0c439..3fb1ac8e 100644 --- a/spring-cloud-zuul-ratelimit-tests/springdata/src/main/resources/application.yml +++ b/spring-cloud-zuul-ratelimit-tests/springdata/src/main/resources/application.yml @@ -103,7 +103,6 @@ zuul: refresh-interval: 60 type: - origin=127.0.0.0/22 - breakOnMatch: true - limit: 1 refresh-interval: 60 type: diff --git a/spring-cloud-zuul-ratelimit-tests/springdata/src/test/java/com/marcosbarbero/tests/it/SpringDataApplicationTestIT.java b/spring-cloud-zuul-ratelimit-tests/springdata/src/test/java/com/marcosbarbero/tests/it/SpringDataApplicationTestIT.java index 154094ab..66b2bf71 100644 --- a/spring-cloud-zuul-ratelimit-tests/springdata/src/test/java/com/marcosbarbero/tests/it/SpringDataApplicationTestIT.java +++ b/spring-cloud-zuul-ratelimit-tests/springdata/src/test/java/com/marcosbarbero/tests/it/SpringDataApplicationTestIT.java @@ -1,25 +1,9 @@ package com.marcosbarbero.tests.it; -import static com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.support.RateLimitConstants.HEADER_LIMIT; -import static com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.support.RateLimitConstants.HEADER_QUOTA; -import static com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.support.RateLimitConstants.HEADER_REMAINING; -import static com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.support.RateLimitConstants.HEADER_REMAINING_QUOTA; -import static com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.support.RateLimitConstants.HEADER_RESET; -import static org.awaitility.Awaitility.await; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.springframework.http.HttpStatus.OK; -import static org.springframework.http.HttpStatus.TOO_MANY_REQUESTS; - import com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.config.RateLimiter; import com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.config.properties.RateLimitProperties; import com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.config.repository.springdata.JpaRateLimiter; import com.marcosbarbero.tests.SpringDataApplication; -import java.util.UUID; -import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -27,6 +11,15 @@ import org.springframework.http.HttpHeaders; import org.springframework.http.ResponseEntity; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static com.marcosbarbero.cloud.autoconfigure.zuul.ratelimit.support.RateLimitConstants.*; +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.*; +import static org.springframework.http.HttpStatus.OK; +import static org.springframework.http.HttpStatus.TOO_MANY_REQUESTS; + /** * @author Marcos Barbero * @since 2017-06-27 @@ -76,7 +69,7 @@ public void testExceedingCapacity() { await().pollDelay(2, TimeUnit.SECONDS).untilAsserted(() -> { final ResponseEntity responseAfterReset = this.restTemplate - .getForEntity("/serviceB", String.class); + .getForEntity("/serviceB", String.class); final HttpHeaders headersAfterReset = responseAfterReset.getHeaders(); assertHeaders(headersAfterReset, key, false, false); assertEquals(OK, responseAfterReset.getStatusCode()); @@ -166,7 +159,7 @@ public void testUsingBreakOnMatchGeneralCaseWithCidr() { private void assertHeaders(HttpHeaders headers, String key, boolean nullable, boolean quotaHeaders) { if (key != null && !key.startsWith("-")) { - key = "-" + key; + key = "-" + key; } String quota = headers.getFirst(HEADER_QUOTA + key); String remainingQuota = headers.getFirst(HEADER_REMAINING_QUOTA + key); diff --git a/spring-cloud-zuul-ratelimit-tests/springdata/src/test/java/com/marcosbarbero/tests/it/SpringDataDenyOriginTestIT.java b/spring-cloud-zuul-ratelimit-tests/springdata/src/test/java/com/marcosbarbero/tests/it/SpringDataDenyOriginTestIT.java new file mode 100644 index 00000000..2797466b --- /dev/null +++ b/spring-cloud-zuul-ratelimit-tests/springdata/src/test/java/com/marcosbarbero/tests/it/SpringDataDenyOriginTestIT.java @@ -0,0 +1,30 @@ +package com.marcosbarbero.tests.it; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.client.TestRestTemplate; +import org.springframework.http.ResponseEntity; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.springframework.http.HttpStatus.NOT_FOUND; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = { + "zuul.ratelimit.default-deny-list.origins[0]=127.0.0.1", + "zuul.ratelimit.default-deny-list.response-status-code=404" + } +) +public class SpringDataDenyOriginTestIT { + + @Autowired + private TestRestTemplate restTemplate; + + @Test + void testDeniedOrigin() { + ResponseEntity response = this.restTemplate.getForEntity("/serviceC", String.class); +// assertEquals(FORBIDDEN, response.getStatusCode()); + assertEquals(NOT_FOUND, response.getStatusCode()); + } + +}