h2: Add a rate limit facility for h/2 RST handling ("Rapid reset" mitigation)

bin/varnishd/http2/cache_http2.h

@@ -193,6 +193,8 @@ struct h2_sess {
 	VTAILQ_HEAD(,h2_req)		txqueue;
 
 	h2_error			error;
+	double				rst_budget;
+	vtim_real			last_rst;
 };
 
 #define ASSERT_RXTHR(h2) do {assert(h2->rxthr == pthread_self());} while(0)

bin/varnishd/http2/cache_http2_proto.c

@@ -46,6 +46,7 @@
 #include "vtcp.h"
 #include "vtim.h"
 
+#define H2_CUSTOM_ERRORS
 #define H2EC1(U,v,r,d) const struct h2_error_s H2CE_##U[1] = {{#U,d,v,0,1,r}};
 #define H2EC2(U,v,r,d) const struct h2_error_s H2SE_##U[1] = {{#U,d,v,1,0,r}};
 #define H2EC3(U,v,r,d) H2EC1(U,v,r,d) H2EC2(U,v,r,d)
@@ -320,9 +321,46 @@ h2_rx_push_promise(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 /**********************************************************************
  */
 
+static h2_error
+h2_rapid_reset(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
+{
+	vtim_real now;
+	vtim_dur d;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	ASSERT_RXTHR(h2);
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
+
+	if (cache_param->h2_rapid_reset_limit == 0)
+		return (0);
+
+	now = VTIM_real();
+	CHECK_OBJ_NOTNULL(r2->req, REQ_MAGIC);
+	AN(r2->req->t_first);
+	if (now - r2->req->t_first > cache_param->h2_rapid_reset)
+		return (0);
+
+	d = now - h2->last_rst;
+	h2->rst_budget += cache_param->h2_rapid_reset_limit * d /
+	    cache_param->h2_rapid_reset_period;
+	h2->rst_budget = vmin_t(double, h2->rst_budget,
+	    cache_param->h2_rapid_reset_limit);
+	h2->last_rst = now;
+
+	if (h2->rst_budget < 1.0) {
+		Lck_Lock(&h2->sess->mtx);
+		VSLb(h2->vsl, SLT_Error, "H2: Hit RST limit. Closing session.");
+		Lck_Unlock(&h2->sess->mtx);
+		return (H2CE_RAPID_RESET);
+	}
+	h2->rst_budget -= 1.0;
+	return (0);
+}
+
 static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_rst_stream(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
+	h2_error h2e;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
@@ -332,8 +370,9 @@ h2_rx_rst_stream(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 		return (H2CE_FRAME_SIZE_ERROR);
 	if (r2 == NULL)
 		return (0);
+	h2e = h2_rapid_reset(wrk, h2, r2);
 	h2_kill_req(wrk, h2, r2, h2_streamerror(vbe32dec(h2->rxf_data)));
-	return (0);
+	return (h2e);
 }
 
 /**********************************************************************

bin/varnishd/http2/cache_http2_session.c

@@ -127,6 +127,9 @@ h2_init_sess(struct sess *sp,
 	h2_local_settings(&h2->local_settings);
 	h2->remote_settings = H2_proto_settings;
 	h2->decode = decode;
+	h2->rst_budget = cache_param->h2_rapid_reset_limit;
+	h2->last_rst = sp->t_open;
+	AZ(isnan(h2->last_rst));
 
 	AZ(VHT_Init(h2->dectbl, h2->local_settings.header_table_size));
 

bin/varnishtest/tests/r03996.vtc

@@ -0,0 +1,51 @@
+varnishtest "h2 rapid reset"
+
+barrier b1 sock 5
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.set debug +syncvsl"
+varnish v1 -cliok "param.set h2_rapid_reset_limit 3"
+varnish v1 -cliok "param.set h2_rapid_reset 5"
+
+varnish v1 -vcl+backend {
+	import vtc;
+
+	sub vcl_recv {
+		vtc.barrier_sync("${b1_sock}");
+	}
+
+} -start
+
+client c1 {
+	stream 0 {
+		rxgoaway
+		expect goaway.err == ENHANCE_YOUR_CALM
+	} -start
+
+	stream 1 {
+		txreq
+		txrst
+	} -run
+	stream 3 {
+		txreq
+		txrst
+	} -run
+	stream 5 {
+		txreq
+		txrst
+	} -run
+	stream 7 {
+		txreq
+		txrst
+	} -run
+
+	barrier b1 sync
+	stream 0 -wait
+} -run
+
+varnish v1 -expect sc_rapid_reset == 1

include/tbl/h2_error.h

@@ -147,5 +147,17 @@ H2_ERROR(
 	/* descr */	"Use HTTP/1.1 for the request"
 )
 
+#ifdef H2_CUSTOM_ERRORS
+H2_ERROR(
+       /* name */      RAPID_RESET,
+       /* val */       11, /* ENHANCE_YOUR_CALM */
+       /* types */     1,
+       /* reason */    SC_RAPID_RESET,
+       /* descr */     "http/2 rapid reset detected"
+)
+
+#  undef H2_CUSTOM_ERRORS
+#endif
+
 #undef H2_ERROR
 /*lint -restore */

include/tbl/params.h

@@ -1257,6 +1257,47 @@ PARAM_SIMPLE(
 	"HTTP2 maximum size of an uncompressed header list."
 )
 
+PARAM_SIMPLE(
+	/* name */	h2_rapid_reset,
+	/* typ */	timeout,
+	/* min */	"0.000",
+	/* max */	NULL,
+	/* def */	"1.000",
+	/* units */	"seconds",
+	/* descr */
+       "The upper threshold for how rapid an http/2 RST has to come for "
+       "it to be treated as suspect and subjected to the rate limits "
+       "specified by h2_rapid_reset_limit and h2_rapid_reset_period.",
+	/* flags */	EXPERIMENTAL,
+)
+
+PARAM_SIMPLE(
+	/* name */	h2_rapid_reset_limit,
+	/* typ */	uint,
+	/* min */	"0",
+	/* max */	NULL,
+	/* def */	"0",
+	/* units */	NULL,
+	/* descr */
+	"HTTP2 RST Allowance.\n"
+	"Specifies the maximum number of allowed stream resets issued by\n"
+	"a client over a time period before the connection is closed.\n"
+	"Setting this parameter to 0 disables the limit.",
+	/* flags */	EXPERIMENTAL,
+)
+
+PARAM_SIMPLE(
+	/* name */	h2_rapid_reset_period,
+	/* typ */	timeout,
+	/* min */	"1.000",
+	/* max */	NULL,
+	/* def */	"60.000",
+	/* units */	"seconds",
+	/* descr */
+	"HTTP2 sliding window duration for h2_rapid_reset_limit.",
+	/* flags */	EXPERIMENTAL|WIZARD,
+)
+
 /*--------------------------------------------------------------------
  * Memory pool parameters
  */

include/tbl/sess_close.h

@@ -50,6 +50,7 @@ SESS_CLOSE(PIPE_OVERFLOW, pipe_overflow,1,	"Session pipe overflow")
 SESS_CLOSE(RANGE_SHORT,   range_short,	1,	"Insufficient data for range")
 SESS_CLOSE(REQ_HTTP20,	  req_http20,	1,	"HTTP2 not accepted")
 SESS_CLOSE(VCL_FAILURE,	  vcl_failure,	1,	"VCL failure")
+SESS_CLOSE(RAPID_RESET,	  rapid_reset,  1,      "HTTP2 rapid reset")
 #undef SESS_CLOSE
 
 /*lint -restore */

lib/libvsc/VSC_main.vsc

@@ -634,6 +634,14 @@
 
 	Number of session closes with Error VCL_FAILURE (VCL failure)
 
+.. varnish_vsc:: sc_rapid_reset
+       :level: diag
+       :oneliner:      Session Err RAPID_RESET
+
+       Number of times we failed an http/2 session because it hit its
+       configured limits for the number of permitted rapid stream
+       resets.
+
 .. varnish_vsc:: client_resp_500
 	:level: diag
 	:group: wrk