Skip to content

Commit

Permalink
Add support for TLS
Browse files Browse the repository at this point in the history
Signed-off-by: Bipul Adhikari <[email protected]>
  • Loading branch information
bipuladh committed Oct 28, 2024
1 parent 568dcb4 commit b0c9bcd
Show file tree
Hide file tree
Showing 11 changed files with 272 additions and 21 deletions.
9 changes: 6 additions & 3 deletions cmd/manager/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ func main() {
enableAdmissionWebhooks bool
ctx = context.Background()
cfg = util.NewConfig()
enableTLS bool
)
flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
Expand All @@ -91,6 +92,7 @@ func main() {
flag.StringVar(&cfg.Namespace, "namespace", cfg.Namespace, "Namespace where the CSIAddons pod is deployed")
flag.BoolVar(&enableAdmissionWebhooks, "enable-admission-webhooks", false, "[DEPRECATED] Enable the admission webhooks")
flag.BoolVar(&showVersion, "version", false, "Print Version details")
flag.BoolVar(&enableTLS, "tls", true, "Enable TLS(enabled by default)")
opts := zap.Options{
Development: true,
TimeEncoder: zapcore.ISO8601TimeEncoder,
Expand Down Expand Up @@ -146,9 +148,10 @@ func main() {
MaxConcurrentReconciles: cfg.MaxConcurrentReconciles,
}
if err = (&controllers.CSIAddonsNodeReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ConnPool: connPool,
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ConnPool: connPool,
EnableTLS: enableTLS,
}).SetupWithManager(mgr, ctrlOptions); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "CSIAddonsNode")
os.Exit(1)
Expand Down
8 changes: 8 additions & 0 deletions config/manager/manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
volumeMounts:
- name: ca-cert
mountPath: /etc/tls/ca
readOnly: true
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
Expand All @@ -64,5 +68,9 @@ spec:
requests:
cpu: 10m
memory: 64Mi
volumes:
- name: tls-secret
secret:
secretName: controller-manager-tls
serviceAccountName: csi-addons-controller-manager
terminationGracePeriodSeconds: 10
16 changes: 16 additions & 0 deletions deploy/controller/csi-addons-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,19 @@ metadata:
data:
"reclaim-space-timeout": "3m"
"max-concurrent-reconciles": "100"
---
apiVersion: v1
kind: Service
metadata:
name: csi-addons-sidecar
# replace the namespace with the namespace where the operator is deployed.
namespace: csi-addons-system
annotations:
service.alpha.openshift.io/serving-cert-secret-name: csi-addons-sidecar-tls
spec:
ports:
- name: grpc
port: 8443
targetPort: 8443
selector:
app: csi-addons-sidecar
8 changes: 8 additions & 0 deletions deploy/controller/setup-controller.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,15 @@ spec:
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
volumeMounts:
- mountPath: /etc/tls/ca
name: ca-cert
readOnly: true
securityContext:
runAsNonRoot: true
serviceAccountName: csi-addons-controller-manager
terminationGracePeriodSeconds: 10
volumes:
- name: tls-secret
secret:
secretName: controller-manager-tls
21 changes: 18 additions & 3 deletions internal/connection/connection.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,15 @@ package connection

import (
"context"
"crypto/x509"
"fmt"
"time"

"github.com/csi-addons/kubernetes-csi-addons/internal/kubernetes/token"
"github.com/csi-addons/spec/lib/go/identity"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/credentials"
"sigs.k8s.io/controller-runtime/pkg/client"
)

// Connection struct consists of to NodeID, DriverName, Capabilities for the controller
Expand All @@ -39,11 +43,22 @@ type Connection struct {

// NewConnection establishes connection with sidecar, fetches capability and returns Connection object
// filled with required information.
func NewConnection(ctx context.Context, endpoint, nodeID, driverName, namespace, podName string) (*Connection, error) {
func NewConnection(ctx context.Context, endpoint, nodeID, driverName, namespace, podName string, client client.Client, enableTLS bool) (*Connection, error) {
opts := []grpc.DialOption{
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithIdleTimeout(time.Duration(0)),
}
opts = append(opts, token.WithServiceAccountToken(client, namespace, "csi-addons-sa"))
if enableTLS {

caFile, caError := token.GetServerCert()
if caError != nil {
panic(fmt.Errorf("failed to get server cert %v", caError))
}
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM([]byte(caFile))
creds := credentials.NewClientTLSFromCert(caCertPool, "")
opts = append(opts, grpc.WithTransportCredentials(creds))
}
cc, err := grpc.NewClient(endpoint, opts...)
if err != nil {
return nil, err
Expand Down
7 changes: 4 additions & 3 deletions internal/controller/csiaddons/csiaddonsnode_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,9 @@ var (
// CSIAddonsNodeReconciler reconciles a CSIAddonsNode object
type CSIAddonsNodeReconciler struct {
client.Client
Scheme *runtime.Scheme
ConnPool *connection.ConnectionPool
Scheme *runtime.Scheme
ConnPool *connection.ConnectionPool
EnableTLS bool
}

//+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
Expand Down Expand Up @@ -120,7 +121,7 @@ func (r *CSIAddonsNodeReconciler) Reconcile(ctx context.Context, req ctrl.Reques
}

logger.Info("Connecting to sidecar")
newConn, err := connection.NewConnection(ctx, endPoint, nodeID, driverName, csiAddonsNode.Namespace, csiAddonsNode.Name)
newConn, err := connection.NewConnection(ctx, endPoint, nodeID, driverName, csiAddonsNode.Namespace, csiAddonsNode.Name, r.Client, r.EnableTLS)
if err != nil {
logger.Error(err, "Failed to establish connection with sidecar")

Expand Down
21 changes: 21 additions & 0 deletions internal/kubernetes/namespace.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package kubernetes

import (
"io"
"os"
)

func GetNamespace() (string, error) {
namespaceFile := "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
file, err := os.Open(namespaceFile)
if err != nil {
return "", err
}
defer file.Close()

data, err := io.ReadAll(file)
if err != nil {
return "", err
}
return string(data), nil
}
151 changes: 151 additions & 0 deletions internal/kubernetes/token/grpc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
/*
Copyright 2024 The Kubernetes-CSI-Addons Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package token

import (
"context"
"fmt"
"io"
"os"

"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
authv1 "k8s.io/api/authentication/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/controller-runtime/pkg/client"
)

type tokenResolver struct {
kubeclient client.Client
namespace string
serviceAccount string

token string
expiration metav1.Time
}

func WithServiceAccountToken(client client.Client, namespace, serviceAccount string) grpc.DialOption {
tr := tokenResolver{
kubeclient: client,
namespace: namespace,
serviceAccount: serviceAccount,
expiration: metav1.Now(),
}

return grpc.WithUnaryInterceptor(tr.addAuthorizationHeader)
}

func (tr *tokenResolver) addAuthorizationHeader(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
token, err := tr.getToken(ctx)
if err != nil {
return err
}

authCtx := metadata.AppendToOutgoingContext(ctx, "Authorization", "Bearer "+token)
return invoker(authCtx, method, req, reply, cc, opts...)
}

func (tr *tokenResolver) getToken(ctx context.Context) (string, error) {
now := metav1.Now()
if tr.expiration.Before(&now) {
// token expired
return tr.refreshToken(ctx)
}

return tr.token, nil
}

func (tr *tokenResolver) refreshToken(ctx context.Context) (string, error) {
treq := &authv1.TokenRequest{
Spec: authv1.TokenRequestSpec{
Audiences: []string{"csi-addons"},
},
}

err := tr.kubeclient.Create(ctx, treq)
if err != nil {
return "", err
}

tr.token = treq.Status.Token
tr.expiration = treq.Status.ExpirationTimestamp

return tr.token, nil
}

func AuthorizationInterceptor(kubeclient kubernetes.Clientset) grpc.UnaryServerInterceptor {
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
if err := authorizeConnection(kubeclient, ctx); err != nil {
return nil, err
}
return handler(ctx, req)
}
}

func authorizeConnection(kubeclient kubernetes.Clientset, ctx context.Context) error {

md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return status.Errorf(codes.Unauthenticated, "missing metadata")
}

authHeader, ok := md["authorization"]
if !ok || len(authHeader) == 0 {
return status.Errorf(codes.Unauthenticated, "missing authorization token")
}

token := authHeader[0]
isValidated, err := validateBearerToken(token, kubeclient)
if !isValidated || err != nil {
return status.Errorf(codes.Unauthenticated, "invalid token")
}
return nil
}

func validateBearerToken(token string, kubeclient kubernetes.Clientset) (bool, error) {
tokenReview := &authv1.TokenReview{
Spec: authv1.TokenReviewSpec{
Token: token,
},
}
result, err := kubeclient.AuthenticationV1().TokenReviews().Create(context.TODO(), tokenReview, metav1.CreateOptions{})
if err != nil {
return false, fmt.Errorf("failed to review token %v", err)
}

if result.Status.Authenticated {
return true, nil
}
return false, nil
}

func GetServerCert() (string, error) {
certFile := "/etc/tls/ca.crt"
file, err := os.Open(certFile)
if err != nil {
return "", err
}
defer file.Close()

data, err := io.ReadAll(file)
if err != nil {
return "", err
}
return string(data), nil
}
30 changes: 23 additions & 7 deletions sidecar/internal/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,13 @@ package server

import (
"errors"
"fmt"
"net"

"github.com/csi-addons/kubernetes-csi-addons/internal/kubernetes/token"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
k8s "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
)

Expand All @@ -38,24 +42,27 @@ type SidecarServer struct {
// URL components to listen on the tcp port
scheme string
endpoint string
client k8s.Clientset

server *grpc.Server
services []SidecarService
server *grpc.Server
services []SidecarService
enableTLS bool
}

// NewSidecarServer create a new SidecarServer on the given IP-address and
// port. If the IP-address is an empty string, the server will listen on all
// available IP-addresses. Only tcp ports are supported.
func NewSidecarServer(ip, port string) *SidecarServer {
func NewSidecarServer(ip string, port int, client k8s.Clientset, enableTLS bool) *SidecarServer {
ss := &SidecarServer{}

if ss.services == nil {
ss.services = make([]SidecarService, 0)
}

ss.scheme = "tcp"
ss.endpoint = ip + ":" + port

ss.endpoint = ip + ":" + fmt.Sprint(port)
ss.client = client
ss.enableTLS = enableTLS
return ss
}

Expand All @@ -69,8 +76,17 @@ func (ss *SidecarServer) RegisterService(svc SidecarService) {
// Init creates the internal gRPC server, and registers the SidecarServices.
// and starts gRPC server.
func (ss *SidecarServer) Start() {
// create the gRPC server and register services
ss.server = grpc.NewServer()
if ss.enableTLS {
creds, err := credentials.NewServerTLSFromFile("/etc/tls/tls.crt", "/etc/tls/tls.key")
if err != nil {
klog.Fatalf("Could not find TLS file: %v", err)
}
// create the gRPC server and register services
ss.server = grpc.NewServer(grpc.UnaryInterceptor(token.AuthorizationInterceptor(ss.client)), grpc.Creds(creds))
}
if !ss.enableTLS {
ss.server = grpc.NewServer(grpc.UnaryInterceptor(token.AuthorizationInterceptor(ss.client)))
}

for _, svc := range ss.services {
svc.RegisterService(ss.server)
Expand Down
Loading

0 comments on commit b0c9bcd

Please sign in to comment.