From 393e82020e9406dada2edb4708225b010c3f42ac Mon Sep 17 00:00:00 2001 From: Siggi Simonarson Date: Wed, 29 Sep 2021 15:09:13 -0400 Subject: [PATCH 1/2] Set a keep-alive enforcement policy (#1068) * Set a keep-alive enforcement policy * Let's go with 10s * Update BUILD * Update comments * Appease checkstyle --- server/util/grpc_server/BUILD | 1 + server/util/grpc_server/grpc_server.go | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/server/util/grpc_server/BUILD b/server/util/grpc_server/BUILD index 246cceea746..9075efab79b 100644 --- a/server/util/grpc_server/BUILD +++ b/server/util/grpc_server/BUILD @@ -15,5 +15,6 @@ go_library( "@io_opentelemetry_go_otel//attribute", "@io_opentelemetry_go_otel_trace//:trace", "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//keepalive", ], ) diff --git a/server/util/grpc_server/grpc_server.go b/server/util/grpc_server/grpc_server.go index addc6dbbbd6..bc2b7cc0783 100644 --- a/server/util/grpc_server/grpc_server.go +++ b/server/util/grpc_server/grpc_server.go @@ -12,6 +12,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" ) @@ -82,5 +83,10 @@ func CommonGRPCServerOptions(env environment.Env) []grpc.ServerOption { grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor), grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor), grpc.MaxRecvMsgSize(env.GetConfigurator().GetGRPCMaxRecvMsgSizeBytes()), + // Set to avoid errors: Bandwidth exhausted HTTP/2 error code: ENHANCE_YOUR_CALM Received Goaway too_many_pings + grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + MinTime: 10 * time.Second, // If a client pings more than once every 10 seconds, terminate the connection + PermitWithoutStream: true, // Allow pings even when there are no active streams + }), } } From e4c90c6a58c5fc52990fafce872c6cab6b870747 Mon Sep 17 00:00:00 2001 From: Brandon Duffany Date: Wed, 29 Sep 2021 17:04:20 -0400 Subject: [PATCH 2/2] Disable redis memory collector (#1069) --- enterprise/server/cmd/server/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enterprise/server/cmd/server/main.go b/enterprise/server/cmd/server/main.go index 1594b7de7d4..0113eb66eff 100644 --- a/enterprise/server/cmd/server/main.go +++ b/enterprise/server/cmd/server/main.go @@ -194,7 +194,7 @@ func main() { redisClient := redisutil.NewClient(redisTarget, healthChecker, "default_redis") realEnv.SetDefaultRedisClient(redisClient) r := redis_cache.NewCache(realEnv.GetDefaultRedisClient(), 0) - realEnv.SetMetricsCollector(r) + // realEnv.SetMetricsCollector(r) realEnv.SetKeyValStore(r) }