Skip to content

Commit

Permalink
csiaddons: add support for TLS
Browse files Browse the repository at this point in the history
Signed-off-by: Bipul Adhikari <[email protected]>
  • Loading branch information
bipuladh committed Nov 13, 2024
1 parent 85fb846 commit 50a7c52
Show file tree
Hide file tree
Showing 5 changed files with 356 additions and 7 deletions.
11 changes: 11 additions & 0 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ func main() {
var probeAddr string
var secureMetrics bool
var enableHTTP2 bool
var enableCertificateGeneration bool
flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
Expand All @@ -66,6 +67,7 @@ func main() {
"If set the metrics endpoint is served securely")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
flag.BoolVar(&enableCertificateGeneration, "enable-cert-generation", false, "If set, CertificateSigningRequests will be used to generate certificates.")
opts := zap.Options{
Development: true,
}
Expand Down Expand Up @@ -122,6 +124,15 @@ func main() {
os.Exit(1)
}

if enableCertificateGeneration {
if err = (&controller.CertsReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "Certificates")
}
}

if err = (&controller.DriverReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Expand Down
134 changes: 134 additions & 0 deletions internal/controller/csr_controller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
/*
Copyright 2024.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
"context"
"crypto/x509"
"encoding/pem"
"errors"
"time"

certv1 "k8s.io/api/certificates/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

//+kubebuilder:rbac:groups=csi.ceph.io,resources=drivers,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=csi.ceph.io,resources=drivers/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=csi.ceph.io,resources=drivers/finalizers,verbs=update
//+kubebuilder:rbac:groups=csi.ceph.io,resources=operatorconfigs,verbs=get;list;watch
//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update
//+kubebuilder:rbac:groups=storage.k8s.io,resources=csidrivers,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete

type CertsReconciler struct {
client.Client
Scheme *runtime.Scheme
privateKey []byte
ctx context.Context
}

func filterCsiCertificateSigningRequests() predicate.Predicate {
return predicate.NewPredicateFuncs(func(object client.Object) bool {
labels := object.GetLabels()
return labels != nil && labels["managed-by"] == "ceph-csi-operator"
})
}

func (r *CertsReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).For(&certv1.CertificateSigningRequest{}).WithEventFilter(filterCsiCertificateSigningRequests()).Complete(r)
}

func (c *CertsReconciler) retrieveSignedCertificate(csr *certv1.CertificateSigningRequest, request ctrl.Request) (ctrl.Result, error) {
if csr.Status.Certificate == nil {
// Retry later if the certificate is not ready yet
return ctrl.Result{RequeueAfter: time.Second * 5}, nil
}
return ctrl.Result{}, nil
}

func (r *CertsReconciler) Reconcile(ctx context.Context, request ctrl.Request) (reconcile.Result, error) {
log := ctrllog.FromContext(ctx)
log.Info("Starting reconcile for Certificate signing requests", "req", request)

csr := &certv1.CertificateSigningRequest{}
if err := r.Get(ctx, request.NamespacedName, csr); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}

if isCSRApproved(csr) {
return r.retrieveSignedCertificate(csr, request)
}

// Validate CSR contents to ensure it meets the requirements
if err := validateCSR(csr); err != nil {
// Log the error, or update the status to indicate failure
return ctrl.Result{}, err
}

// Approve the CSR
csr.Status.Conditions = append(csr.Status.Conditions, certv1.CertificateSigningRequestCondition{
Type: certv1.CertificateApproved,
Status: v1.ConditionTrue,
Reason: "AutoApproved",
Message: "CSR auto-approved by Ceph CSI CSR Controller",
LastUpdateTime: metav1.Now(),
})

if err := r.Status().Update(ctx, csr); err != nil {
return ctrl.Result{}, err
}

return ctrl.Result{}, nil
}

func isCSRApproved(csr *certv1.CertificateSigningRequest) bool {
for _, condition := range csr.Status.Conditions {
if condition.Type == certv1.CertificateApproved {
return true
}
}
return false
}

// validateCSR validates CSR fields for the required certificate attributes
func validateCSR(csr *certv1.CertificateSigningRequest) error {
csrBytes, _ := pem.Decode(csr.Spec.Request)
if csrBytes == nil {
return errors.New("failed to parse CSR PEM")
}
certRequest, err := x509.ParseCertificateRequest(csrBytes.Bytes)
if err != nil {
return err
}

// Validate required fields (e.g., organization, common name, etc.)
if len(certRequest.Subject.Organization) == 0 || certRequest.Subject.CommonName == "" {
return errors.New("CSR is missing required fields")
}

return nil
}
75 changes: 70 additions & 5 deletions internal/controller/driver_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,11 +93,60 @@ type DriverReconciler struct {
type driverReconcile struct {
DriverReconciler

ctx context.Context
log logr.Logger
driver csiv1a1.Driver
driverType DriverType
images map[string]string
ctx context.Context
log logr.Logger
driver csiv1a1.Driver
driverType DriverType
images map[string]string
certificateGenerated bool
}

func createK8sSecret(csrPEM, privKeyPEM []byte, secretName, namespace string) (*corev1.Secret, error) {
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: namespace,
},
Data: map[string][]byte{
"tls.crt": csrPEM, // Store the CSR as the certificate (it can be replaced with an actual cert later)
"tls.key": privKeyPEM,
},
}

return secret, nil
}

func (r *driverReconcile) performCertificateGeneration(Name string, log logr.Logger) error {

if r.isRdbDriver() {
if !r.isCertificateGenerated() {
secretName := "tls-cert"
requestObject, privateKey, err := utils.GetCertificateSigningRequest(r.driver.Name,
Name,
[]string{"rook"}, []string{strings.Join([]string{Name, "*"},
"kubernetes.io/kube-apiserver-client")}, "ceph-csi")
if err != nil {
log.Error(err, "Failed to create certificate signign request object")
return err
}
err = r.Client.Create(r.ctx, requestObject)
if err != nil {
log.Error(err, "Failed to create CertificateSigningRequest resource")
return err
}
secretObject, err := createK8sSecret(requestObject.Spec.Request, privateKey, secretName, r.driver.Namespace)
if err != nil {
log.Error(err, "Failed to create CertificateSigningRequest resource")
return err
}
err = r.Client.Create(r.ctx, secretObject)
if err != nil {
log.Error(err, "Failed to create secret resource")
return err
}
}
}
return nil
}

// SetupWithManager sets up the controller with the Manager.
Expand Down Expand Up @@ -516,6 +565,10 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
log := r.log.WithValues("deploymentName", deploy.Name)
log.Info("Reconciling controller plugin deployment")

if err := r.performCertificateGeneration(deploy.Name, log); err != nil {
return err
}

opResult, err := ctrlutil.CreateOrUpdate(r.ctx, r.Client, deploy, func() error {
if err := ctrlutil.SetControllerReference(&r.driver, deploy, r.Scheme); err != nil {
log.Error(err, "Failed setting an owner reference on deployment")
Expand Down Expand Up @@ -776,6 +829,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""),
utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""),
utils.If(logRotationEnabled, utils.LogFileContainerArg("csi-addons"), ""),
utils.RBDCtrlPluginAddonsSidecarTlsSecretName,
),
),
Ports: []corev1.ContainerPort{
Expand Down Expand Up @@ -929,6 +983,12 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
log := r.log.WithValues("daemonSetName", daemonSet.Name)
log.Info("Reconciling node plugin deployment")

if err := r.performCertificateGeneration(daemonSet.Name, log); err != nil {
return err
}

// if is rbd driver and certificates are required then generate a certificate if not already

opResult, err := ctrlutil.CreateOrUpdate(r.ctx, r.Client, daemonSet, func() error {
if err := ctrlutil.SetControllerReference(&r.driver, daemonSet, r.Scheme); err != nil {
log.Error(err, "Failed setting an owner reference on deployment")
Expand Down Expand Up @@ -1249,6 +1309,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
volumes = append(
volumes,
utils.OidcTokenVolume,
utils.TlsCertsRBDNodePlugin,
)
}
if logRotationEnabled {
Expand Down Expand Up @@ -1314,6 +1375,10 @@ func (r *driverReconcile) reconcileLivenessService() error {
}
}

func (r *driverReconcile) isCertificateGenerated() bool {
return r.certificateGenerated
}

func (r *driverReconcile) isRdbDriver() bool {
return r.driverType == RbdDriverType
}
Expand Down
105 changes: 105 additions & 0 deletions internal/utils/certificates.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
/*
Copyright 2024.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package utils

import (
"crypto"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"fmt"

certv1 "k8s.io/api/certificates/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// GetCertificateSigningRequest generates a Certificate Signing Request (CSR)
// for a given common name, organization, and Subject Alternative Names (SANs).
//
// Parameters:
// - commonName: The common name (CN) for the certificate request.
// - organization: A slice of organization names (O) for the certificate request.
// - san: A slice of DNS names to be included as Subject Alternative Names.
// - signerName: The name of the signer (e.g., kubernetes.io/kube-apiserver-client).
// - usages: A slice of KeyUsage values specifying the intended usages of the certificate.
//
// Returns:
// - *certv1.CertificateSigningRequest: A populated CSR object ready for submission.
// - []byte: The private key in PEM-encoded format corresponding to the CSR.
// - error: An error if there was an issue generating the CSR or private key.
//
// The function generates an ECDSA private key, creates a CSR template with the provided
// details, and encodes the private key to PEM format. The resulting CSR is returned along
// with the private key PEM data.
//
// Example usage:
//
// csr, privateKeyPEM, err := GetCertificateSigningRequest("example.com", []string{"ExampleOrg"}, []string{"example.com"}, "kubernetes.io/kubelet-serving", []certv1.KeyUsage{certv1.UsageServerAuth})
func GetCertificateSigningRequest(csrRequestName, commonName string, organization, san []string, signerName string) (*certv1.CertificateSigningRequest, []byte, error) {

privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return nil, nil, fmt.Errorf("failed generating private key")
}
csrTemplate := x509.CertificateRequest{
Subject: pkix.Name{
CommonName: commonName,
Organization: organization,
},
DNSNames: san,
}

createCertificateRequest, err := x509.CreateCertificateRequest(rand.Reader, &csrTemplate, privateKey)
if err != nil {
return nil, nil, fmt.Errorf("failed to create CSR: %v", err)
}

privateKeyPEM, err := encodePrivateKeyToPEM(privateKey)
if err != nil {
return nil, nil, fmt.Errorf("failed to encode private key: %v", err)
}
return (&certv1.CertificateSigningRequest{
ObjectMeta: metav1.ObjectMeta{
Name: csrRequestName,
Labels: map[string]string{
"managed-by": "ceph-csi-operator",
},
},
Spec: certv1.CertificateSigningRequestSpec{
Request: createCertificateRequest,
SignerName: signerName,
Usages: []certv1.KeyUsage{
certv1.UsageDigitalSignature,
certv1.UsageKeyEncipherment,
certv1.UsageServerAuth,
},
},
}), privateKeyPEM, nil

}

func encodePrivateKeyToPEM(priv crypto.PrivateKey) ([]byte, error) {
privBytes, err := x509.MarshalECPrivateKey(priv.(*ecdsa.PrivateKey))
if err != nil {
return nil, fmt.Errorf("failed to marshal private key: %v", err)
}
privPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: privBytes})
return privPEM, nil
}
Loading

0 comments on commit 50a7c52

Please sign in to comment.