From bdc2e10aeec606ff69f563edcbe6bca5ed080f8a Mon Sep 17 00:00:00 2001 From: Bipul Adhikari Date: Wed, 13 Nov 2024 17:21:56 +0545 Subject: [PATCH] Adds support for TLS encryption between manager and addons Signed-off-by: Bipul Adhikari --- README.md | 13 ++ cmd/manager/main.go | 13 +- docs/deploy-controller.md | 1 + internal/connection/connection.go | 28 +++- .../csiaddons/csiaddonsnode_controller.go | 12 +- internal/kubernetes/token/grpc.go | 120 ++++++++++++++++++ sidecar/internal/server/server.go | 32 ++++- sidecar/main.go | 5 +- 8 files changed, 208 insertions(+), 16 deletions(-) create mode 100644 internal/kubernetes/token/grpc.go diff --git a/README.md b/README.md index adefd1d39..883e5549f 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,15 @@ the CSI-nodeplugin containers. The side-car registers itself by creating a `CSIAddonsNode` CR that the CSI-Addons Controller can use to connect to the side-car and execute operations. +### Enabling TLS for side-car + +To enable TLS for the sidecar, a certificate and key are required. The certificate +should be configured to allow verification on the manager via a shared Certificate Authority (CA). + +- **Certificate Requirements**: Ensure that Subject Alternative Names (SANs) are configured to + include pod DNS names for proper verification. +- **Token Validation**: Grant access to the TokenReview API to enable bearer token validation. + ### `csi-addons` executable The `csi-addons` executable can be used to call CSI-Addons operations against a @@ -64,6 +73,10 @@ By listing the `CSIAddonsNode` CRs, the CSI-Addons Controller knows how to connect to the side-cars. By checking the supported capabilities of the side-cars, it can decide where to execute operations that the user requested. +### Enabling TLS for manager + +When deploying the manager set `enable-tls` flag to true. + ### Installation Refer to the [installation guide](docs/deploy-controller.md) for more details. diff --git a/cmd/manager/main.go b/cmd/manager/main.go index df218544b..ccde2d2c8 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -77,6 +77,8 @@ func main() { enableAdmissionWebhooks bool ctx = context.Background() cfg = util.NewConfig() + enableTLS bool + skipInsecureVerify bool ) flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -92,6 +94,8 @@ func main() { flag.BoolVar(&enableAdmissionWebhooks, "enable-admission-webhooks", false, "[DEPRECATED] Enable the admission webhooks") flag.BoolVar(&showVersion, "version", false, "Print Version details") flag.StringVar(&cfg.SchedulePrecedence, "schedule-precedence", "", "The order of precedence in which schedule of reclaimspace and keyrotation is considered. Possible values are sc-only") + flag.BoolVar(&enableTLS, "enable-tls", false, "Enable TLS (disabled by default)") + flag.BoolVar(&skipInsecureVerify, "insecure-skip-tls-verify", false, "skip server certificate verification") opts := zap.Options{ Development: true, TimeEncoder: zapcore.ISO8601TimeEncoder, @@ -145,16 +149,17 @@ func main() { setupLog.Error(err, "unable to start manager") os.Exit(1) } - connPool := connection.NewConnectionPool() ctrlOptions := controller.Options{ MaxConcurrentReconciles: cfg.MaxConcurrentReconciles, } if err = (&controllers.CSIAddonsNodeReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - ConnPool: connPool, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + ConnPool: connPool, + EnableTLS: enableTLS, + SkipInsecureVerify: skipInsecureVerify, }).SetupWithManager(mgr, ctrlOptions); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CSIAddonsNode") os.Exit(1) diff --git a/docs/deploy-controller.md b/docs/deploy-controller.md index 8d8bd1ad9..659bc841b 100644 --- a/docs/deploy-controller.md +++ b/docs/deploy-controller.md @@ -13,6 +13,7 @@ The CSI-Addons Controller can be deployed by different ways: | `--leader-elect` | `false` | Enable leader election for controller manager. | | `--reclaim-space-timeout` | `3m` | Timeout for reclaimspace operation | | `--max-concurrent-reconciles` | 100 | Maximum number of concurrent reconciles | +| `--enable-tls` | false | Maximum number of concurrent reconciles | > Note: Some of the above configuration options can also be configured using [`"csi-addons-config"` configmap](./csi-addons-config.md). diff --git a/internal/connection/connection.go b/internal/connection/connection.go index 0b24f0210..63ad199be 100644 --- a/internal/connection/connection.go +++ b/internal/connection/connection.go @@ -18,10 +18,17 @@ package connection import ( "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" "time" + "github.com/csi-addons/kubernetes-csi-addons/internal/kubernetes/token" + "github.com/csi-addons/spec/lib/go/identity" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" ) @@ -39,10 +46,27 @@ type Connection struct { // NewConnection establishes connection with sidecar, fetches capability and returns Connection object // filled with required information. -func NewConnection(ctx context.Context, endpoint, nodeID, driverName, namespace, podName string) (*Connection, error) { +func NewConnection(ctx context.Context, endpoint, nodeID, driverName, namespace, podName string, enableTLS, skipInsecureVerify bool) (*Connection, error) { opts := []grpc.DialOption{ grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithIdleTimeout(time.Duration(0)), + } + if enableTLS { + opts = append(opts, token.WithServiceAccountToken()) + + caFile, caError := token.GetCACert() + if caError != nil { + return nil, fmt.Errorf("failed to get server cert %w", caError) + } + caCertPool := x509.NewCertPool() + if !caCertPool.AppendCertsFromPEM([]byte(caFile)) { + return nil, errors.New("failed to append CA cert") + } + tlsConfig := &tls.Config{ + RootCAs: caCertPool, // The CA certificates to verify the server + InsecureSkipVerify: skipInsecureVerify, + } + creds := credentials.NewTLS(tlsConfig) + opts = append(opts, grpc.WithTransportCredentials(creds)) } cc, err := grpc.NewClient(endpoint, opts...) if err != nil { diff --git a/internal/controller/csiaddons/csiaddonsnode_controller.go b/internal/controller/csiaddons/csiaddonsnode_controller.go index 15b80f4df..e4280bfcb 100644 --- a/internal/controller/csiaddons/csiaddonsnode_controller.go +++ b/internal/controller/csiaddons/csiaddonsnode_controller.go @@ -47,8 +47,10 @@ var ( // CSIAddonsNodeReconciler reconciles a CSIAddonsNode object type CSIAddonsNodeReconciler struct { client.Client - Scheme *runtime.Scheme - ConnPool *connection.ConnectionPool + Scheme *runtime.Scheme + ConnPool *connection.ConnectionPool + EnableTLS bool + SkipInsecureVerify bool } //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch @@ -121,7 +123,7 @@ func (r *CSIAddonsNodeReconciler) Reconcile(ctx context.Context, req ctrl.Reques } logger.Info("Connecting to sidecar") - newConn, err := connection.NewConnection(ctx, endPoint, nodeID, driverName, csiAddonsNode.Namespace, csiAddonsNode.Name) + newConn, err := connection.NewConnection(ctx, endPoint, nodeID, driverName, csiAddonsNode.Namespace, csiAddonsNode.Name, r.EnableTLS, r.SkipInsecureVerify) if err != nil { logger.Error(err, "Failed to establish connection with sidecar") @@ -211,6 +213,10 @@ func (r *CSIAddonsNodeReconciler) resolveEndpoint(ctx context.Context, rawURL st if err != nil { return "", "", err } + if r.EnableTLS { + // We need to use this name to accept certificates signed for pods + return podname, podname + "." + namespace + ".pod" + ":" + port, nil + } pod := &corev1.Pod{} err = r.Client.Get(ctx, client.ObjectKey{ diff --git a/internal/kubernetes/token/grpc.go b/internal/kubernetes/token/grpc.go new file mode 100644 index 000000000..a0b3f3877 --- /dev/null +++ b/internal/kubernetes/token/grpc.go @@ -0,0 +1,120 @@ +/* +Copyright 2024 The Kubernetes-CSI-Addons Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package token + +import ( + "context" + "fmt" + "io" + "os" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + authv1 "k8s.io/api/authentication/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const bearerPrefix = "Bearer " + +func WithServiceAccountToken() grpc.DialOption { + return grpc.WithUnaryInterceptor(addAuthorizationHeader) +} + +func addAuthorizationHeader(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + token, err := getToken() + if err != nil { + return err + } + + authCtx := metadata.AppendToOutgoingContext(ctx, "Authorization", "Bearer "+token) + return invoker(authCtx, method, req, reply, cc, opts...) +} + +func getToken() (string, error) { + return readFile("/var/run/secrets/kubernetes.io/serviceaccount/token") +} + +func AuthorizationInterceptor(kubeclient kubernetes.Clientset) grpc.UnaryServerInterceptor { + return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + if err := authorizeConnection(ctx, kubeclient); err != nil { + return nil, err + } + return handler(ctx, req) + } +} + +func authorizeConnection(ctx context.Context, kubeclient kubernetes.Clientset) error { + + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return status.Error(codes.Unauthenticated, "missing metadata") + } + + authHeader, ok := md["authorization"] + if !ok || len(authHeader) == 0 { + return status.Error(codes.Unauthenticated, "missing authorization token") + } + + token := authHeader[0] + isValidated, err := validateBearerToken(ctx, token, kubeclient) + if !isValidated || (err != nil) { + return status.Error(codes.Unauthenticated, fmt.Sprint("invalid token: %w", err)) + } + return nil +} + +func parseToken(authHeader string) string { + return strings.TrimPrefix(authHeader, bearerPrefix) +} + +func validateBearerToken(ctx context.Context, token string, kubeclient kubernetes.Clientset) (bool, error) { + tokenReview := &authv1.TokenReview{ + Spec: authv1.TokenReviewSpec{ + Token: parseToken(token), + }, + } + result, err := kubeclient.AuthenticationV1().TokenReviews().Create(ctx, tokenReview, metav1.CreateOptions{}) + if err != nil { + return false, fmt.Errorf("failed to review token %w", err) + } + + if result.Status.Authenticated { + return true, nil + } + return false, nil +} + +func readFile(filePath string) (string, error) { + file, err := os.Open(filePath) + if err != nil { + return "", err + } + defer file.Close() + + data, err := io.ReadAll(file) + if err != nil { + return "", err + } + return string(data), nil +} + +func GetCACert() (string, error) { + return readFile("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") +} diff --git a/sidecar/internal/server/server.go b/sidecar/internal/server/server.go index 112554683..874e74b16 100644 --- a/sidecar/internal/server/server.go +++ b/sidecar/internal/server/server.go @@ -20,7 +20,11 @@ import ( "errors" "net" + "github.com/csi-addons/kubernetes-csi-addons/internal/kubernetes/token" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + k8s "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" ) @@ -38,15 +42,19 @@ type SidecarServer struct { // URL components to listen on the tcp port scheme string endpoint string + client *k8s.Clientset - server *grpc.Server - services []SidecarService + server *grpc.Server + services []SidecarService + enableTLS bool + tlsCertFile string + tlsKeyFile string } // NewSidecarServer create a new SidecarServer on the given IP-address and // port. If the IP-address is an empty string, the server will listen on all // available IP-addresses. Only tcp ports are supported. -func NewSidecarServer(ip, port string) *SidecarServer { +func NewSidecarServer(ip, port string, client *k8s.Clientset, enableTLS bool, tlsCertFile, tlsKeyFile string) *SidecarServer { ss := &SidecarServer{} if ss.services == nil { @@ -55,7 +63,10 @@ func NewSidecarServer(ip, port string) *SidecarServer { ss.scheme = "tcp" ss.endpoint = ip + ":" + port - + ss.client = client + ss.enableTLS = enableTLS + ss.tlsCertFile = tlsCertFile + ss.tlsKeyFile = tlsKeyFile return ss } @@ -69,8 +80,17 @@ func (ss *SidecarServer) RegisterService(svc SidecarService) { // Init creates the internal gRPC server, and registers the SidecarServices. // and starts gRPC server. func (ss *SidecarServer) Start() { - // create the gRPC server and register services - ss.server = grpc.NewServer() + if ss.enableTLS { + creds, err := credentials.NewServerTLSFromFile(ss.tlsCertFile, ss.tlsKeyFile) + if err != nil { + klog.Fatalf("failed to load TLS certificate and key: %v", err) + } + // create the gRPC server and register services + ss.server = grpc.NewServer(grpc.UnaryInterceptor(token.AuthorizationInterceptor(*ss.client)), grpc.Creds(creds)) + } + if !ss.enableTLS { + ss.server = grpc.NewServer() + } for _, svc := range ss.services { svc.RegisterService(ss.server) diff --git a/sidecar/main.go b/sidecar/main.go index 24fd3fe84..1d4938111 100644 --- a/sidecar/main.go +++ b/sidecar/main.go @@ -56,6 +56,9 @@ func main() { leaderElectionLeaseDuration = flag.Duration("leader-election-lease-duration", 15*time.Second, "Duration, in seconds, that non-leader candidates will wait to force acquire leadership. Defaults to 15 seconds.") leaderElectionRenewDeadline = flag.Duration("leader-election-renew-deadline", 10*time.Second, "Duration, in seconds, that the acting leader will retry refreshing leadership before giving up. Defaults to 10 seconds.") leaderElectionRetryPeriod = flag.Duration("leader-election-retry-period", 5*time.Second, "Duration, in seconds, the LeaderElector clients should wait between tries of actions. Defaults to 5 seconds.") + enableTLS = flag.Bool("enable-tls", false, "Enable TLS (disabled by default)") + tlsCertFile = flag.String("tls-cert-file", "/etc/tls/tls.crt", "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert)") + tlsKeyFile = flag.String("tls-private-key-file", "/etc/tls/tls.key", "File containing the default x509 private key matching --tls-cert-file.") ) klog.InitFlags(nil) @@ -110,7 +113,7 @@ func main() { klog.Fatalf("Failed to create csiaddonsnode: %v", err) } - sidecarServer := server.NewSidecarServer(*controllerIP, *controllerPort) + sidecarServer := server.NewSidecarServer(*controllerIP, *controllerPort, kubeClient, *enableTLS, *tlsCertFile, *tlsKeyFile) sidecarServer.RegisterService(service.NewIdentityServer(csiClient.GetGRPCClient())) sidecarServer.RegisterService(service.NewReclaimSpaceServer(csiClient.GetGRPCClient(), kubeClient, *stagingPath)) sidecarServer.RegisterService(service.NewNetworkFenceServer(csiClient.GetGRPCClient(), kubeClient))