Skip to content

Commit

Permalink
Adds support for TLS encryption between manager and addons
Browse files Browse the repository at this point in the history
Signed-off-by: Bipul Adhikari <[email protected]>
  • Loading branch information
bipuladh committed Nov 14, 2024
1 parent 6821a14 commit bdc2e10
Show file tree
Hide file tree
Showing 8 changed files with 208 additions and 16 deletions.
13 changes: 13 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,15 @@ the CSI-nodeplugin containers. The side-car registers itself by creating a
`CSIAddonsNode` CR that the CSI-Addons Controller can use to connect to the
side-car and execute operations.

### Enabling TLS for side-car

To enable TLS for the sidecar, a certificate and key are required. The certificate
should be configured to allow verification on the manager via a shared Certificate Authority (CA).

- **Certificate Requirements**: Ensure that Subject Alternative Names (SANs) are configured to
include pod DNS names for proper verification.
- **Token Validation**: Grant access to the TokenReview API to enable bearer token validation.

### `csi-addons` executable

The `csi-addons` executable can be used to call CSI-Addons operations against a
Expand All @@ -64,6 +73,10 @@ By listing the `CSIAddonsNode` CRs, the CSI-Addons Controller knows how to
connect to the side-cars. By checking the supported capabilities of the
side-cars, it can decide where to execute operations that the user requested.

### Enabling TLS for manager

When deploying the manager set `enable-tls` flag to true.

### Installation

Refer to the [installation guide](docs/deploy-controller.md) for more details.
Expand Down
13 changes: 9 additions & 4 deletions cmd/manager/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ func main() {
enableAdmissionWebhooks bool
ctx = context.Background()
cfg = util.NewConfig()
enableTLS bool
skipInsecureVerify bool
)
flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
Expand All @@ -92,6 +94,8 @@ func main() {
flag.BoolVar(&enableAdmissionWebhooks, "enable-admission-webhooks", false, "[DEPRECATED] Enable the admission webhooks")
flag.BoolVar(&showVersion, "version", false, "Print Version details")
flag.StringVar(&cfg.SchedulePrecedence, "schedule-precedence", "", "The order of precedence in which schedule of reclaimspace and keyrotation is considered. Possible values are sc-only")
flag.BoolVar(&enableTLS, "enable-tls", false, "Enable TLS (disabled by default)")
flag.BoolVar(&skipInsecureVerify, "insecure-skip-tls-verify", false, "skip server certificate verification")
opts := zap.Options{
Development: true,
TimeEncoder: zapcore.ISO8601TimeEncoder,
Expand Down Expand Up @@ -145,16 +149,17 @@ func main() {
setupLog.Error(err, "unable to start manager")
os.Exit(1)
}

connPool := connection.NewConnectionPool()

ctrlOptions := controller.Options{
MaxConcurrentReconciles: cfg.MaxConcurrentReconciles,
}
if err = (&controllers.CSIAddonsNodeReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ConnPool: connPool,
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ConnPool: connPool,
EnableTLS: enableTLS,
SkipInsecureVerify: skipInsecureVerify,
}).SetupWithManager(mgr, ctrlOptions); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "CSIAddonsNode")
os.Exit(1)
Expand Down
1 change: 1 addition & 0 deletions docs/deploy-controller.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ The CSI-Addons Controller can be deployed by different ways:
| `--leader-elect` | `false` | Enable leader election for controller manager. |
| `--reclaim-space-timeout` | `3m` | Timeout for reclaimspace operation |
| `--max-concurrent-reconciles` | 100 | Maximum number of concurrent reconciles |
| `--enable-tls` | false | Maximum number of concurrent reconciles |

> Note: Some of the above configuration options can also be configured using [`"csi-addons-config"` configmap](./csi-addons-config.md).
Expand Down
28 changes: 26 additions & 2 deletions internal/connection/connection.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,17 @@ package connection

import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"time"

"github.com/csi-addons/kubernetes-csi-addons/internal/kubernetes/token"

"github.com/csi-addons/spec/lib/go/identity"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"
)

Expand All @@ -39,10 +46,27 @@ type Connection struct {

// NewConnection establishes connection with sidecar, fetches capability and returns Connection object
// filled with required information.
func NewConnection(ctx context.Context, endpoint, nodeID, driverName, namespace, podName string) (*Connection, error) {
func NewConnection(ctx context.Context, endpoint, nodeID, driverName, namespace, podName string, enableTLS, skipInsecureVerify bool) (*Connection, error) {
opts := []grpc.DialOption{
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithIdleTimeout(time.Duration(0)),
}
if enableTLS {
opts = append(opts, token.WithServiceAccountToken())

caFile, caError := token.GetCACert()
if caError != nil {
return nil, fmt.Errorf("failed to get server cert %w", caError)
}
caCertPool := x509.NewCertPool()
if !caCertPool.AppendCertsFromPEM([]byte(caFile)) {
return nil, errors.New("failed to append CA cert")
}
tlsConfig := &tls.Config{
RootCAs: caCertPool, // The CA certificates to verify the server
InsecureSkipVerify: skipInsecureVerify,
}
creds := credentials.NewTLS(tlsConfig)
opts = append(opts, grpc.WithTransportCredentials(creds))
}
cc, err := grpc.NewClient(endpoint, opts...)
if err != nil {
Expand Down
12 changes: 9 additions & 3 deletions internal/controller/csiaddons/csiaddonsnode_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,10 @@ var (
// CSIAddonsNodeReconciler reconciles a CSIAddonsNode object
type CSIAddonsNodeReconciler struct {
client.Client
Scheme *runtime.Scheme
ConnPool *connection.ConnectionPool
Scheme *runtime.Scheme
ConnPool *connection.ConnectionPool
EnableTLS bool
SkipInsecureVerify bool
}

//+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
Expand Down Expand Up @@ -121,7 +123,7 @@ func (r *CSIAddonsNodeReconciler) Reconcile(ctx context.Context, req ctrl.Reques
}

logger.Info("Connecting to sidecar")
newConn, err := connection.NewConnection(ctx, endPoint, nodeID, driverName, csiAddonsNode.Namespace, csiAddonsNode.Name)
newConn, err := connection.NewConnection(ctx, endPoint, nodeID, driverName, csiAddonsNode.Namespace, csiAddonsNode.Name, r.EnableTLS, r.SkipInsecureVerify)
if err != nil {
logger.Error(err, "Failed to establish connection with sidecar")

Expand Down Expand Up @@ -211,6 +213,10 @@ func (r *CSIAddonsNodeReconciler) resolveEndpoint(ctx context.Context, rawURL st
if err != nil {
return "", "", err
}
if r.EnableTLS {
// We need to use this name to accept certificates signed for pods
return podname, podname + "." + namespace + ".pod" + ":" + port, nil
}

pod := &corev1.Pod{}
err = r.Client.Get(ctx, client.ObjectKey{
Expand Down
120 changes: 120 additions & 0 deletions internal/kubernetes/token/grpc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
/*
Copyright 2024 The Kubernetes-CSI-Addons Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package token

import (
"context"
"fmt"
"io"
"os"
"strings"

"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
authv1 "k8s.io/api/authentication/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)

const bearerPrefix = "Bearer "

func WithServiceAccountToken() grpc.DialOption {
return grpc.WithUnaryInterceptor(addAuthorizationHeader)
}

func addAuthorizationHeader(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
token, err := getToken()
if err != nil {
return err
}

authCtx := metadata.AppendToOutgoingContext(ctx, "Authorization", "Bearer "+token)
return invoker(authCtx, method, req, reply, cc, opts...)
}

func getToken() (string, error) {
return readFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
}

func AuthorizationInterceptor(kubeclient kubernetes.Clientset) grpc.UnaryServerInterceptor {
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
if err := authorizeConnection(ctx, kubeclient); err != nil {
return nil, err
}
return handler(ctx, req)
}
}

func authorizeConnection(ctx context.Context, kubeclient kubernetes.Clientset) error {

md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return status.Error(codes.Unauthenticated, "missing metadata")
}

authHeader, ok := md["authorization"]
if !ok || len(authHeader) == 0 {
return status.Error(codes.Unauthenticated, "missing authorization token")
}

token := authHeader[0]
isValidated, err := validateBearerToken(ctx, token, kubeclient)
if !isValidated || (err != nil) {
return status.Error(codes.Unauthenticated, fmt.Sprint("invalid token: %w", err))
}
return nil
}

func parseToken(authHeader string) string {
return strings.TrimPrefix(authHeader, bearerPrefix)
}

func validateBearerToken(ctx context.Context, token string, kubeclient kubernetes.Clientset) (bool, error) {
tokenReview := &authv1.TokenReview{
Spec: authv1.TokenReviewSpec{
Token: parseToken(token),
},
}
result, err := kubeclient.AuthenticationV1().TokenReviews().Create(ctx, tokenReview, metav1.CreateOptions{})
if err != nil {
return false, fmt.Errorf("failed to review token %w", err)
}

if result.Status.Authenticated {
return true, nil
}
return false, nil
}

func readFile(filePath string) (string, error) {
file, err := os.Open(filePath)
if err != nil {
return "", err
}
defer file.Close()

data, err := io.ReadAll(file)
if err != nil {
return "", err
}
return string(data), nil
}

func GetCACert() (string, error) {
return readFile("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
}
32 changes: 26 additions & 6 deletions sidecar/internal/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,11 @@ import (
"errors"
"net"

"github.com/csi-addons/kubernetes-csi-addons/internal/kubernetes/token"

"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
k8s "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
)

Expand All @@ -38,15 +42,19 @@ type SidecarServer struct {
// URL components to listen on the tcp port
scheme string
endpoint string
client *k8s.Clientset

server *grpc.Server
services []SidecarService
server *grpc.Server
services []SidecarService
enableTLS bool
tlsCertFile string
tlsKeyFile string
}

// NewSidecarServer create a new SidecarServer on the given IP-address and
// port. If the IP-address is an empty string, the server will listen on all
// available IP-addresses. Only tcp ports are supported.
func NewSidecarServer(ip, port string) *SidecarServer {
func NewSidecarServer(ip, port string, client *k8s.Clientset, enableTLS bool, tlsCertFile, tlsKeyFile string) *SidecarServer {
ss := &SidecarServer{}

if ss.services == nil {
Expand All @@ -55,7 +63,10 @@ func NewSidecarServer(ip, port string) *SidecarServer {

ss.scheme = "tcp"
ss.endpoint = ip + ":" + port

ss.client = client
ss.enableTLS = enableTLS
ss.tlsCertFile = tlsCertFile
ss.tlsKeyFile = tlsKeyFile
return ss
}

Expand All @@ -69,8 +80,17 @@ func (ss *SidecarServer) RegisterService(svc SidecarService) {
// Init creates the internal gRPC server, and registers the SidecarServices.
// and starts gRPC server.
func (ss *SidecarServer) Start() {
// create the gRPC server and register services
ss.server = grpc.NewServer()
if ss.enableTLS {
creds, err := credentials.NewServerTLSFromFile(ss.tlsCertFile, ss.tlsKeyFile)
if err != nil {
klog.Fatalf("failed to load TLS certificate and key: %v", err)
}
// create the gRPC server and register services
ss.server = grpc.NewServer(grpc.UnaryInterceptor(token.AuthorizationInterceptor(*ss.client)), grpc.Creds(creds))
}
if !ss.enableTLS {
ss.server = grpc.NewServer()
}

for _, svc := range ss.services {
svc.RegisterService(ss.server)
Expand Down
5 changes: 4 additions & 1 deletion sidecar/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ func main() {
leaderElectionLeaseDuration = flag.Duration("leader-election-lease-duration", 15*time.Second, "Duration, in seconds, that non-leader candidates will wait to force acquire leadership. Defaults to 15 seconds.")
leaderElectionRenewDeadline = flag.Duration("leader-election-renew-deadline", 10*time.Second, "Duration, in seconds, that the acting leader will retry refreshing leadership before giving up. Defaults to 10 seconds.")
leaderElectionRetryPeriod = flag.Duration("leader-election-retry-period", 5*time.Second, "Duration, in seconds, the LeaderElector clients should wait between tries of actions. Defaults to 5 seconds.")
enableTLS = flag.Bool("enable-tls", false, "Enable TLS (disabled by default)")
tlsCertFile = flag.String("tls-cert-file", "/etc/tls/tls.crt", "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert)")
tlsKeyFile = flag.String("tls-private-key-file", "/etc/tls/tls.key", "File containing the default x509 private key matching --tls-cert-file.")
)
klog.InitFlags(nil)

Expand Down Expand Up @@ -110,7 +113,7 @@ func main() {
klog.Fatalf("Failed to create csiaddonsnode: %v", err)
}

sidecarServer := server.NewSidecarServer(*controllerIP, *controllerPort)
sidecarServer := server.NewSidecarServer(*controllerIP, *controllerPort, kubeClient, *enableTLS, *tlsCertFile, *tlsKeyFile)
sidecarServer.RegisterService(service.NewIdentityServer(csiClient.GetGRPCClient()))
sidecarServer.RegisterService(service.NewReclaimSpaceServer(csiClient.GetGRPCClient(), kubeClient, *stagingPath))
sidecarServer.RegisterService(service.NewNetworkFenceServer(csiClient.GetGRPCClient(), kubeClient))
Expand Down

0 comments on commit bdc2e10

Please sign in to comment.