From a6a5cd5438936ac383d2313a9b3640e32b2eeac8 Mon Sep 17 00:00:00 2001 From: Dag Haavi Finstad Date: Thu, 12 Oct 2023 10:50:55 +0200 Subject: [PATCH 1/3] h2: Add a rate limit facility for h/2 RST handling This adds parameters h2_rst_allowance and h2_rst_allowance_period, which govern the rate of which we allow clients to reset h/2 streams. If the limit is exceeded the connection is closed. Mitigates: #3996 --- bin/varnishd/http2/cache_http2.h | 2 + bin/varnishd/http2/cache_http2_proto.c | 35 ++++++++++++++++- bin/varnishd/http2/cache_http2_session.c | 3 ++ bin/varnishtest/tests/r03996.vtc | 49 ++++++++++++++++++++++++ include/tbl/params.h | 28 ++++++++++++++ 5 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 bin/varnishtest/tests/r03996.vtc diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index eb6e8f6a9f3..9457c99f2dd 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -193,6 +193,8 @@ struct h2_sess { VTAILQ_HEAD(,h2_req) txqueue; h2_error error; + double rst_budget; + vtim_real last_rst; }; #define ASSERT_RXTHR(h2) do {assert(h2->rxthr == pthread_self());} while(0) diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index c62a8c429fa..2720824980b 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -320,9 +320,41 @@ h2_rx_push_promise(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) /********************************************************************** */ +static h2_error +h2_rapid_reset(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) +{ + vtim_real now; + vtim_dur d; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + ASSERT_RXTHR(h2); + CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); + + if (cache_param->h2_rapid_reset_limit == 0) + return (0); + + now = VTIM_real(); + d = now - h2->last_rst; + h2->rst_budget += cache_param->h2_rapid_reset_limit * d / + cache_param->h2_rapid_reset_period; + h2->rst_budget = vmin_t(double, h2->rst_budget, + cache_param->h2_rapid_reset_limit); + h2->last_rst = now; + + if (h2->rst_budget < 1.0) { + Lck_Lock(&h2->sess->mtx); + VSLb(h2->vsl, SLT_Error, "H2: Hit RST limit. Closing session."); + Lck_Unlock(&h2->sess->mtx); + return (H2CE_ENHANCE_YOUR_CALM); + } + h2->rst_budget -= 1.0; + return (0); +} + static h2_error v_matchproto_(h2_rxframe_f) h2_rx_rst_stream(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) { + h2_error h2e; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); ASSERT_RXTHR(h2); @@ -332,8 +364,9 @@ h2_rx_rst_stream(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) return (H2CE_FRAME_SIZE_ERROR); if (r2 == NULL) return (0); + h2e = h2_rapid_reset(wrk, h2, r2); h2_kill_req(wrk, h2, r2, h2_streamerror(vbe32dec(h2->rxf_data))); - return (0); + return (h2e); } /********************************************************************** diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index e3e3c116c8b..672b40d6435 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -127,6 +127,9 @@ h2_init_sess(struct sess *sp, h2_local_settings(&h2->local_settings); h2->remote_settings = H2_proto_settings; h2->decode = decode; + h2->rst_budget = cache_param->h2_rapid_reset_limit; + h2->last_rst = sp->t_open; + AZ(isnan(h2->last_rst)); AZ(VHT_Init(h2->dectbl, h2->local_settings.header_table_size)); diff --git a/bin/varnishtest/tests/r03996.vtc b/bin/varnishtest/tests/r03996.vtc new file mode 100644 index 00000000000..5864c837f00 --- /dev/null +++ b/bin/varnishtest/tests/r03996.vtc @@ -0,0 +1,49 @@ +varnishtest "h2 rapid reset" + +barrier b1 sock 5 + +server s1 { + rxreq + txresp +} -start + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.set debug +syncvsl" +varnish v1 -cliok "param.set h2_rapid_reset_limit 3" + +varnish v1 -vcl+backend { + import vtc; + + sub vcl_recv { + vtc.barrier_sync("${b1_sock}"); + } + +} -start + +client c1 { + stream 0 { + rxgoaway + expect goaway.err == ENHANCE_YOUR_CALM + } -start + + stream 1 { + txreq + txrst + } -run + stream 3 { + txreq + txrst + } -run + stream 5 { + txreq + txrst + } -run + stream 7 { + txreq + txrst + } -run + + barrier b1 sync + stream 0 -wait +} -run + diff --git a/include/tbl/params.h b/include/tbl/params.h index cb7b7f5aa0b..cbea1bc31b6 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1257,6 +1257,34 @@ PARAM_SIMPLE( "HTTP2 maximum size of an uncompressed header list." ) +PARAM_SIMPLE( + /* name */ h2_rapid_reset_limit, + /* typ */ uint, + /* min */ "0", + /* max */ NULL, + /* def */ "0", + /* units */ NULL, + /* descr */ + "HTTP2 RST Allowance.\n" + "Specifies the maximum number of allowed stream resets issued by\n" + "a client over a time period before the connection is closed.\n" + "Setting this parameter to 0 disables the limit.", + /* flags */ EXPERIMENTAL, +) + +PARAM_SIMPLE( + /* name */ h2_rapid_reset_period, + /* typ */ timeout, + /* min */ "1.000", + /* max */ NULL, + /* def */ "60.000", + /* units */ "seconds", + /* descr */ + "HTTP2 sliding window duration for h2_rapid_reset_limit.", + /* flags */ EXPERIMENTAL|WIZARD, +) + + /*-------------------------------------------------------------------- * Memory pool parameters */ From 7236bc0990acb478ce01eafbf232b28ebfec2a7a Mon Sep 17 00:00:00 2001 From: Dag Haavi Finstad Date: Tue, 17 Oct 2023 11:20:11 +0200 Subject: [PATCH 2/3] Introduce RAPID_RESET as a sess_close reason --- bin/varnishd/http2/cache_http2_proto.c | 3 ++- bin/varnishtest/tests/r03996.vtc | 1 + include/tbl/h2_error.h | 12 ++++++++++++ include/tbl/sess_close.h | 1 + lib/libvsc/VSC_main.vsc | 8 ++++++++ 5 files changed, 24 insertions(+), 1 deletion(-) diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 2720824980b..a464a45085c 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -46,6 +46,7 @@ #include "vtcp.h" #include "vtim.h" +#define H2_CUSTOM_ERRORS #define H2EC1(U,v,r,d) const struct h2_error_s H2CE_##U[1] = {{#U,d,v,0,1,r}}; #define H2EC2(U,v,r,d) const struct h2_error_s H2SE_##U[1] = {{#U,d,v,1,0,r}}; #define H2EC3(U,v,r,d) H2EC1(U,v,r,d) H2EC2(U,v,r,d) @@ -345,7 +346,7 @@ h2_rapid_reset(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) Lck_Lock(&h2->sess->mtx); VSLb(h2->vsl, SLT_Error, "H2: Hit RST limit. Closing session."); Lck_Unlock(&h2->sess->mtx); - return (H2CE_ENHANCE_YOUR_CALM); + return (H2CE_RAPID_RESET); } h2->rst_budget -= 1.0; return (0); diff --git a/bin/varnishtest/tests/r03996.vtc b/bin/varnishtest/tests/r03996.vtc index 5864c837f00..6feef496303 100644 --- a/bin/varnishtest/tests/r03996.vtc +++ b/bin/varnishtest/tests/r03996.vtc @@ -47,3 +47,4 @@ client c1 { stream 0 -wait } -run +varnish v1 -expect sc_rapid_reset == 1 diff --git a/include/tbl/h2_error.h b/include/tbl/h2_error.h index 2cd06f01107..c6c6bed8918 100644 --- a/include/tbl/h2_error.h +++ b/include/tbl/h2_error.h @@ -147,5 +147,17 @@ H2_ERROR( /* descr */ "Use HTTP/1.1 for the request" ) +#ifdef H2_CUSTOM_ERRORS +H2_ERROR( + /* name */ RAPID_RESET, + /* val */ 11, /* ENHANCE_YOUR_CALM */ + /* types */ 1, + /* reason */ SC_RAPID_RESET, + /* descr */ "http/2 rapid reset detected" +) + +# undef H2_CUSTOM_ERRORS +#endif + #undef H2_ERROR /*lint -restore */ diff --git a/include/tbl/sess_close.h b/include/tbl/sess_close.h index b91c93951c9..f15d34a64aa 100644 --- a/include/tbl/sess_close.h +++ b/include/tbl/sess_close.h @@ -50,6 +50,7 @@ SESS_CLOSE(PIPE_OVERFLOW, pipe_overflow,1, "Session pipe overflow") SESS_CLOSE(RANGE_SHORT, range_short, 1, "Insufficient data for range") SESS_CLOSE(REQ_HTTP20, req_http20, 1, "HTTP2 not accepted") SESS_CLOSE(VCL_FAILURE, vcl_failure, 1, "VCL failure") +SESS_CLOSE(RAPID_RESET, rapid_reset, 1, "HTTP2 rapid reset") #undef SESS_CLOSE /*lint -restore */ diff --git a/lib/libvsc/VSC_main.vsc b/lib/libvsc/VSC_main.vsc index a4049fccb25..1f63b45a41b 100644 --- a/lib/libvsc/VSC_main.vsc +++ b/lib/libvsc/VSC_main.vsc @@ -634,6 +634,14 @@ Number of session closes with Error VCL_FAILURE (VCL failure) +.. varnish_vsc:: sc_rapid_reset + :level: diag + :oneliner: Session Err RAPID_RESET + + Number of times we failed an http/2 session because it hit its + configured limits for the number of permitted rapid stream + resets. + .. varnish_vsc:: client_resp_500 :level: diag :group: wrk From e5c5abf6102e39d4c10079c4d1fdc42338b92efb Mon Sep 17 00:00:00 2001 From: Dag Haavi Finstad Date: Tue, 17 Oct 2023 11:27:44 +0200 Subject: [PATCH 3/3] Add param h2_rapid_reset Only RST frames received earlier than this duration will be considered rapid. --- bin/varnishd/http2/cache_http2_proto.c | 5 +++++ bin/varnishtest/tests/r03996.vtc | 1 + include/tbl/params.h | 15 ++++++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index a464a45085c..50504c02138 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -335,6 +335,11 @@ h2_rapid_reset(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) return (0); now = VTIM_real(); + CHECK_OBJ_NOTNULL(r2->req, REQ_MAGIC); + AN(r2->req->t_first); + if (now - r2->req->t_first > cache_param->h2_rapid_reset) + return (0); + d = now - h2->last_rst; h2->rst_budget += cache_param->h2_rapid_reset_limit * d / cache_param->h2_rapid_reset_period; diff --git a/bin/varnishtest/tests/r03996.vtc b/bin/varnishtest/tests/r03996.vtc index 6feef496303..3fee3706c35 100644 --- a/bin/varnishtest/tests/r03996.vtc +++ b/bin/varnishtest/tests/r03996.vtc @@ -10,6 +10,7 @@ server s1 { varnish v1 -cliok "param.set feature +http2" varnish v1 -cliok "param.set debug +syncvsl" varnish v1 -cliok "param.set h2_rapid_reset_limit 3" +varnish v1 -cliok "param.set h2_rapid_reset 5" varnish v1 -vcl+backend { import vtc; diff --git a/include/tbl/params.h b/include/tbl/params.h index cbea1bc31b6..530b60c39e9 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1257,6 +1257,20 @@ PARAM_SIMPLE( "HTTP2 maximum size of an uncompressed header list." ) +PARAM_SIMPLE( + /* name */ h2_rapid_reset, + /* typ */ timeout, + /* min */ "0.000", + /* max */ NULL, + /* def */ "1.000", + /* units */ "seconds", + /* descr */ + "The upper threshold for how rapid an http/2 RST has to come for " + "it to be treated as suspect and subjected to the rate limits " + "specified by h2_rapid_reset_limit and h2_rapid_reset_period.", + /* flags */ EXPERIMENTAL, +) + PARAM_SIMPLE( /* name */ h2_rapid_reset_limit, /* typ */ uint, @@ -1284,7 +1298,6 @@ PARAM_SIMPLE( /* flags */ EXPERIMENTAL|WIZARD, ) - /*-------------------------------------------------------------------- * Memory pool parameters */