Skip to content

Commit

Permalink
Adds support for TLS and token reviews
Browse files Browse the repository at this point in the history
Signed-off-by: Bipul Adhikari <[email protected]>
  • Loading branch information
bipuladh committed Nov 11, 2024
1 parent 568dcb4 commit 16a68cd
Show file tree
Hide file tree
Showing 9 changed files with 272 additions and 20 deletions.
10 changes: 6 additions & 4 deletions cmd/manager/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ func main() {
enableAdmissionWebhooks bool
ctx = context.Background()
cfg = util.NewConfig()
enableTLS bool
)
flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
Expand All @@ -91,6 +92,7 @@ func main() {
flag.StringVar(&cfg.Namespace, "namespace", cfg.Namespace, "Namespace where the CSIAddons pod is deployed")
flag.BoolVar(&enableAdmissionWebhooks, "enable-admission-webhooks", false, "[DEPRECATED] Enable the admission webhooks")
flag.BoolVar(&showVersion, "version", false, "Print Version details")
flag.BoolVar(&enableTLS, "tls", true, "Enable TLS(enabled by default)")
opts := zap.Options{
Development: true,
TimeEncoder: zapcore.ISO8601TimeEncoder,
Expand Down Expand Up @@ -139,16 +141,16 @@ func main() {
setupLog.Error(err, "unable to start manager")
os.Exit(1)
}

connPool := connection.NewConnectionPool()

ctrlOptions := controller.Options{
MaxConcurrentReconciles: cfg.MaxConcurrentReconciles,
}
if err = (&controllers.CSIAddonsNodeReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ConnPool: connPool,
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ConnPool: connPool,
EnableTLS: enableTLS,
}).SetupWithManager(mgr, ctrlOptions); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "CSIAddonsNode")
os.Exit(1)
Expand Down
11 changes: 11 additions & 0 deletions config/manager/manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
volumeMounts:
- name: certs
mountPath: /etc/tls
readOnly: true
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
Expand All @@ -64,5 +68,12 @@ spec:
requests:
cpu: 10m
memory: 64Mi
volumes:
- name: certs
configMap:
name: openshift-service-ca.crt
items:
- key: service-ca.crt
path: tls.crt
serviceAccountName: csi-addons-controller-manager
terminationGracePeriodSeconds: 10
11 changes: 11 additions & 0 deletions deploy/controller/setup-controller.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,18 @@ spec:
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
volumeMounts:
- mountPath: /etc/tls
name: certs
readOnly: true
securityContext:
runAsNonRoot: true
serviceAccountName: csi-addons-controller-manager
terminationGracePeriodSeconds: 10
volumes:
- configMap:
items:
- key: service-ca.crt
path: tls.crt
name: openshift-service-ca.crt
name: certs
28 changes: 25 additions & 3 deletions internal/connection/connection.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,15 @@ package connection

import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"time"

"github.com/csi-addons/kubernetes-csi-addons/internal/kubernetes/token"
"github.com/csi-addons/spec/lib/go/identity"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/credentials"
)

// Connection struct consists of to NodeID, DriverName, Capabilities for the controller
Expand All @@ -39,11 +43,29 @@ type Connection struct {

// NewConnection establishes connection with sidecar, fetches capability and returns Connection object
// filled with required information.
func NewConnection(ctx context.Context, endpoint, nodeID, driverName, namespace, podName string) (*Connection, error) {
func NewConnection(ctx context.Context, endpoint, nodeID, driverName, namespace, podName string, enableTLS bool) (*Connection, error) {
opts := []grpc.DialOption{
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithIdleTimeout(time.Duration(0)),
}

opts = append(opts, token.WithServiceAccountToken())
if enableTLS {

caFile, caError := token.GetCACert()
if caError != nil {
return nil, (fmt.Errorf("failed to get server cert %v", caError))
}
caCertPool := x509.NewCertPool()
if !caCertPool.AppendCertsFromPEM([]byte(caFile)) {
return nil, fmt.Errorf("failed to append CA cert")
}
tlsConfig := &tls.Config{
RootCAs: caCertPool, // The CA certificates to verify the server
InsecureSkipVerify: true,
}
creds := credentials.NewTLS(tlsConfig)
opts = append(opts, grpc.WithTransportCredentials(creds))
}
cc, err := grpc.NewClient(endpoint, opts...)
if err != nil {
return nil, err
Expand Down
7 changes: 4 additions & 3 deletions internal/controller/csiaddons/csiaddonsnode_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,9 @@ var (
// CSIAddonsNodeReconciler reconciles a CSIAddonsNode object
type CSIAddonsNodeReconciler struct {
client.Client
Scheme *runtime.Scheme
ConnPool *connection.ConnectionPool
Scheme *runtime.Scheme
ConnPool *connection.ConnectionPool
EnableTLS bool
}

//+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
Expand Down Expand Up @@ -120,7 +121,7 @@ func (r *CSIAddonsNodeReconciler) Reconcile(ctx context.Context, req ctrl.Reques
}

logger.Info("Connecting to sidecar")
newConn, err := connection.NewConnection(ctx, endPoint, nodeID, driverName, csiAddonsNode.Namespace, csiAddonsNode.Name)
newConn, err := connection.NewConnection(ctx, endPoint, nodeID, driverName, csiAddonsNode.Namespace, csiAddonsNode.Name, r.EnableTLS)
if err != nil {
logger.Error(err, "Failed to establish connection with sidecar")

Expand Down
36 changes: 36 additions & 0 deletions internal/kubernetes/namespace.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
Copyright 2024 The Kubernetes-CSI-Addons Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubernetes

import (
"io"
"os"
)

func GetNamespace() (string, error) {
namespaceFile := "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
file, err := os.Open(namespaceFile)
if err != nil {
return "", err
}
defer file.Close()

data, err := io.ReadAll(file)
if err != nil {
return "", err
}
return string(data), nil
}
142 changes: 142 additions & 0 deletions internal/kubernetes/token/grpc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
/*
Copyright 2024 The Kubernetes-CSI-Addons Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package token

import (
"context"
"fmt"
"io"
"os"
"strings"

"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
authv1 "k8s.io/api/authentication/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)

func WithServiceAccountToken() grpc.DialOption {
return grpc.WithUnaryInterceptor(addAuthorizationHeader)
}

func addAuthorizationHeader(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
token, err := getToken()
if err != nil {
return err
}

authCtx := metadata.AppendToOutgoingContext(ctx, "Authorization", "Bearer "+token)
return invoker(authCtx, method, req, reply, cc, opts...)
}

func getToken() (string, error) {

const tokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token"

file, err := os.Open(tokenPath)
if err != nil {
return "", err
}
defer file.Close()

data, err := io.ReadAll(file)
if err != nil {
return "", err
}
return string(data), nil
}

func AuthorizationInterceptor(kubeclient kubernetes.Clientset) grpc.UnaryServerInterceptor {
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
if err := authorizeConnection(ctx, kubeclient); err != nil {
return nil, err
}
return handler(ctx, req)
}
}

func authorizeConnection(ctx context.Context, kubeclient kubernetes.Clientset) error {

md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return status.Errorf(codes.Unauthenticated, "missing metadata")
}

authHeader, ok := md["authorization"]
if !ok || len(authHeader) == 0 {
return status.Errorf(codes.Unauthenticated, "missing authorization token")
}

token := authHeader[0]
isValidated, err := validateBearerToken(ctx, token, kubeclient)
if !isValidated || (err != nil) {
fmt.Printf("Error after validation %v", err)
return status.Errorf(codes.Unauthenticated, "invalid token")
}
return nil
}

func removeBearer(authHeader string) string {
// Check if the Authorization header starts with "Bearer"
if strings.HasPrefix(authHeader, "Bearer ") {
// Remove the "Bearer " part and return the token
return strings.TrimPrefix(authHeader, "Bearer ")
}
// If it doesn't start with "Bearer", return the original header
return authHeader
}

func validateBearerToken(ctx context.Context, token string, kubeclient kubernetes.Clientset) (bool, error) {
tokenReview := &authv1.TokenReview{
Spec: authv1.TokenReviewSpec{
Token: removeBearer(token),
},
}
result, err := kubeclient.AuthenticationV1().TokenReviews().Create(ctx, tokenReview, metav1.CreateOptions{})
fmt.Printf("result %v", result)
fmt.Printf("token: %v", token)
if err != nil {
fmt.Printf("err in k8s client: %v", err)
return false, fmt.Errorf("failed to review token %v", err)
}

if result.Status.Authenticated {
return true, nil
}
return false, nil
}

func readFile(filePath string) (string, error) {
file, err := os.Open(filePath)
if err != nil {
return "", err
}
defer file.Close()

data, err := io.ReadAll(file)
if err != nil {
return "", err
}
return string(data), nil
}

func GetCACert() (string, error) {
caCertFile := "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
return readFile(caCertFile)
}
30 changes: 23 additions & 7 deletions sidecar/internal/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,13 @@ package server

import (
"errors"
"fmt"
"net"

"github.com/csi-addons/kubernetes-csi-addons/internal/kubernetes/token"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
k8s "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
)

Expand All @@ -38,24 +42,27 @@ type SidecarServer struct {
// URL components to listen on the tcp port
scheme string
endpoint string
client k8s.Clientset

server *grpc.Server
services []SidecarService
server *grpc.Server
services []SidecarService
enableTLS bool
}

// NewSidecarServer create a new SidecarServer on the given IP-address and
// port. If the IP-address is an empty string, the server will listen on all
// available IP-addresses. Only tcp ports are supported.
func NewSidecarServer(ip, port string) *SidecarServer {
func NewSidecarServer(ip string, port int, client k8s.Clientset, enableTLS bool) *SidecarServer {
ss := &SidecarServer{}

if ss.services == nil {
ss.services = make([]SidecarService, 0)
}

ss.scheme = "tcp"
ss.endpoint = ip + ":" + port

ss.endpoint = ip + ":" + fmt.Sprint(port)
ss.client = client
ss.enableTLS = enableTLS
return ss
}

Expand All @@ -69,8 +76,17 @@ func (ss *SidecarServer) RegisterService(svc SidecarService) {
// Init creates the internal gRPC server, and registers the SidecarServices.
// and starts gRPC server.
func (ss *SidecarServer) Start() {
// create the gRPC server and register services
ss.server = grpc.NewServer()
if ss.enableTLS {
creds, err := credentials.NewServerTLSFromFile("/etc/tls/tls.crt", "/etc/tls/tls.key")
if err != nil {
klog.Fatalf("Could not find TLS file: %v", err)
}
// create the gRPC server and register services
ss.server = grpc.NewServer(grpc.UnaryInterceptor(token.AuthorizationInterceptor(ss.client)), grpc.Creds(creds))
}
if !ss.enableTLS {
ss.server = grpc.NewServer(grpc.UnaryInterceptor(token.AuthorizationInterceptor(ss.client)))
}

for _, svc := range ss.services {
svc.RegisterService(ss.server)
Expand Down
Loading

0 comments on commit 16a68cd

Please sign in to comment.