diff --git a/cmd/main.go b/cmd/main.go index 97228a5e..fd2b1714 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -57,6 +57,7 @@ func main() { var probeAddr string var secureMetrics bool var enableHTTP2 bool + var enableCertificateGeneration bool flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, @@ -66,6 +67,7 @@ func main() { "If set the metrics endpoint is served securely") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") + flag.BoolVar(&enableCertificateGeneration, "enable-cert-generation", false, "If set, CertificateSigningRequests will be used to generate certificates.") opts := zap.Options{ Development: true, } @@ -122,6 +124,15 @@ func main() { os.Exit(1) } + if enableCertificateGeneration { + if err = (&controller.CertsReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Certificates") + } + } + if err = (&controller.DriverReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index d60de3bc..cf55ab12 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,5 +1,7 @@ resources: - manager.yaml +- manager_role.yaml +- manager_role_binding.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization patches: diff --git a/config/manager/managaer_role.yaml b/config/manager/managaer_role.yaml new file mode 100644 index 00000000..12002542 --- /dev/null +++ b/config/manager/managaer_role.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: manager-role +rules: + # Permissions for CSR management + - apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests"] + verbs: ["create", "update", "get", "list", "approve"] + + # Permissions for Secret management + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "update", "get", "list"] diff --git a/config/manager/manager_role_binding.yaml b/config/manager/manager_role_binding.yaml new file mode 100644 index 00000000..2ba57d9b --- /dev/null +++ b/config/manager/manager_role_binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: manager-role-binding +subjects: + - kind: ServiceAccount + name: controller-manageer +roleRef: + kind: Role + name: manager-role + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/internal/controller/csr_controller.go b/internal/controller/csr_controller.go new file mode 100644 index 00000000..d16dfacf --- /dev/null +++ b/internal/controller/csr_controller.go @@ -0,0 +1,132 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "crypto/x509" + "encoding/pem" + "errors" + "time" + + certv1 "k8s.io/api/certificates/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +//+kubebuilder:rbac:groups=csi.ceph.io,resources=drivers,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=csi.ceph.io,resources=drivers/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=csi.ceph.io,resources=drivers/finalizers,verbs=update +//+kubebuilder:rbac:groups=csi.ceph.io,resources=operatorconfigs,verbs=get;list;watch +//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update +//+kubebuilder:rbac:groups=storage.k8s.io,resources=csidrivers,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete + +type CertsReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +func filterCsiCertificateSigningRequests() predicate.Predicate { + return predicate.NewPredicateFuncs(func(object client.Object) bool { + labels := object.GetLabels() + return labels != nil && labels["managed-by"] == "ceph-csi-operator" + }) +} + +func (r *CertsReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr).For(&certv1.CertificateSigningRequest{}).WithEventFilter(filterCsiCertificateSigningRequests()).Complete(r) +} + +func (c *CertsReconciler) retrieveSignedCertificate(csr *certv1.CertificateSigningRequest) (ctrl.Result, error) { + if csr.Status.Certificate == nil { + // Retry later if the certificate is not ready yet + return ctrl.Result{RequeueAfter: time.Second * 5}, nil + } + return ctrl.Result{}, nil +} + +func (r *CertsReconciler) Reconcile(ctx context.Context, request ctrl.Request) (reconcile.Result, error) { + log := ctrllog.FromContext(ctx) + log.Info("Starting reconcile for Certificate signing requests", "req", request) + + csr := &certv1.CertificateSigningRequest{} + if err := r.Get(ctx, request.NamespacedName, csr); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if isCSRApproved(csr) { + return r.retrieveSignedCertificate(csr) + } + + // Validate CSR contents to ensure it meets the requirements + if err := validateCSR(csr); err != nil { + // Log the error, or update the status to indicate failure + return ctrl.Result{}, err + } + + // Approve the CSR + csr.Status.Conditions = append(csr.Status.Conditions, certv1.CertificateSigningRequestCondition{ + Type: certv1.CertificateApproved, + Status: v1.ConditionTrue, + Reason: "AutoApproved", + Message: "CSR auto-approved by Ceph CSI CSR Controller", + LastUpdateTime: metav1.Now(), + }) + + if err := r.Status().Update(ctx, csr); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +func isCSRApproved(csr *certv1.CertificateSigningRequest) bool { + for _, condition := range csr.Status.Conditions { + if condition.Type == certv1.CertificateApproved { + return true + } + } + return false +} + +// validateCSR validates CSR fields for the required certificate attributes +func validateCSR(csr *certv1.CertificateSigningRequest) error { + csrBytes, _ := pem.Decode(csr.Spec.Request) + if csrBytes == nil { + return errors.New("failed to parse CSR PEM") + } + certRequest, err := x509.ParseCertificateRequest(csrBytes.Bytes) + if err != nil { + return err + } + + // Validate required fields (e.g., organization, common name, etc.) + if len(certRequest.Subject.Organization) == 0 || certRequest.Subject.CommonName == "" { + return errors.New("CSR is missing required fields") + } + + return nil +} diff --git a/internal/controller/driver_controller.go b/internal/controller/driver_controller.go index e5a1ac1e..a1eeb382 100644 --- a/internal/controller/driver_controller.go +++ b/internal/controller/driver_controller.go @@ -86,18 +86,68 @@ var nameRegExp, _ = regexp.Compile(fmt.Sprintf( // DriverReconciler reconciles a Driver object type DriverReconciler struct { client.Client - Scheme *runtime.Scheme + Scheme *runtime.Scheme + enableTLS bool } // A local reconcile object tied to a single reconcile iteration type driverReconcile struct { DriverReconciler - ctx context.Context - log logr.Logger - driver csiv1a1.Driver - driverType DriverType - images map[string]string + ctx context.Context + log logr.Logger + driver csiv1a1.Driver + driverType DriverType + images map[string]string + certificateGenerated bool +} + +func createK8sSecret(csrPEM, privKeyPEM []byte, secretName, namespace string) (*corev1.Secret, error) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: namespace, + }, + Data: map[string][]byte{ + "tls.crt": csrPEM, // Store the CSR as the certificate (it can be replaced with an actual cert later) + "tls.key": privKeyPEM, + }, + } + + return secret, nil +} + +func (r *driverReconcile) performCertificateGeneration(Name string, log logr.Logger) error { + + if r.isRdbDriver() { + if !r.isCertificateGenerated() { + secretName := "tls-cert" + requestObject, privateKey, err := utils.GetCertificateSigningRequest(r.driver.Name, + Name, + []string{"rook"}, []string{strings.Join([]string{Name, "*"}, + "kubernetes.io/kube-apiserver-client")}, "ceph-csi") + if err != nil { + log.Error(err, "Failed to create certificate signign request object") + return err + } + err = r.Client.Create(r.ctx, requestObject) + if err != nil { + log.Error(err, "Failed to create CertificateSigningRequest resource") + return err + } + secretObject, err := createK8sSecret(requestObject.Spec.Request, privateKey, secretName, r.driver.Namespace) + if err != nil { + log.Error(err, "Failed to create CertificateSigningRequest resource") + return err + } + err = r.Client.Create(r.ctx, secretObject) + if err != nil { + log.Error(err, "Failed to create secret resource") + return err + } + } + } + return nil } // SetupWithManager sets up the controller with the Manager. @@ -516,6 +566,10 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { log := r.log.WithValues("deploymentName", deploy.Name) log.Info("Reconciling controller plugin deployment") + if err := r.performCertificateGeneration(deploy.Name, log); err != nil { + return err + } + opResult, err := ctrlutil.CreateOrUpdate(r.ctx, r.Client, deploy, func() error { if err := ctrlutil.SetControllerReference(&r.driver, deploy, r.Scheme); err != nil { log.Error(err, "Failed setting an owner reference on deployment") @@ -794,6 +848,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { if logRotationEnabled { mounts = append(mounts, utils.LogsDirVolumeMount) } + if r.enableTLS { + mounts = append(mounts, utils.RBDCtrlPluginTLSCertVolumeMount) + } return mounts }), Resources: ptr.Deref( @@ -908,6 +965,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.LogRotateDirVolumeName(r.driver.Name), ) } + if r.enableTLS { + volumes = append(volumes, utils.TlsCertsRBDCtrlPluginVolume) + } return volumes }), }, @@ -929,6 +989,12 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { log := r.log.WithValues("daemonSetName", daemonSet.Name) log.Info("Reconciling node plugin deployment") + if err := r.performCertificateGeneration(daemonSet.Name, log); err != nil { + return err + } + + // if is rbd driver and certificates are required then generate a certificate if not already + opResult, err := ctrlutil.CreateOrUpdate(r.ctx, r.Client, daemonSet, func() error { if err := ctrlutil.SetControllerReference(&r.driver, daemonSet, r.Scheme); err != nil { log.Error(err, "Failed setting an owner reference on deployment") @@ -1129,6 +1195,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""), utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""), utils.If(logRotationEnabled, utils.LogFileContainerArg("csi-addons"), ""), + utils.If(r.enableTLS, "--enable-tls=true", ""), }, ), Ports: []corev1.ContainerPort{ @@ -1147,6 +1214,9 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { if logRotationEnabled { mounts = append(mounts, utils.LogsDirVolumeMount) } + if r.enableTLS { + mounts = append(mounts, utils.RBDNodePluginTLSCertVolumeMount) + } return mounts }), Resources: ptr.Deref( @@ -1250,6 +1320,9 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { volumes, utils.OidcTokenVolume, ) + if r.enableTLS { + volumes = append(volumes, utils.TlsCertsRBDNodePluginVolume) + } } if logRotationEnabled { logHostPath := cmp.Or(logRotationSpec.LogHostPath, defaultLogHostPath) @@ -1314,6 +1387,10 @@ func (r *driverReconcile) reconcileLivenessService() error { } } +func (r *driverReconcile) isCertificateGenerated() bool { + return r.certificateGenerated +} + func (r *driverReconcile) isRdbDriver() bool { return r.driverType == RbdDriverType } diff --git a/internal/utils/certificates.go b/internal/utils/certificates.go new file mode 100644 index 00000000..731ac530 --- /dev/null +++ b/internal/utils/certificates.go @@ -0,0 +1,105 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + + certv1 "k8s.io/api/certificates/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// GetCertificateSigningRequest generates a Certificate Signing Request (CSR) +// for a given common name, organization, and Subject Alternative Names (SANs). +// +// Parameters: +// - commonName: The common name (CN) for the certificate request. +// - organization: A slice of organization names (O) for the certificate request. +// - san: A slice of DNS names to be included as Subject Alternative Names. +// - signerName: The name of the signer (e.g., kubernetes.io/kube-apiserver-client). +// - usages: A slice of KeyUsage values specifying the intended usages of the certificate. +// +// Returns: +// - *certv1.CertificateSigningRequest: A populated CSR object ready for submission. +// - []byte: The private key in PEM-encoded format corresponding to the CSR. +// - error: An error if there was an issue generating the CSR or private key. +// +// The function generates an ECDSA private key, creates a CSR template with the provided +// details, and encodes the private key to PEM format. The resulting CSR is returned along +// with the private key PEM data. +// +// Example usage: +// +// csr, privateKeyPEM, err := GetCertificateSigningRequest("example.com", []string{"ExampleOrg"}, []string{"example.com"}, "kubernetes.io/kubelet-serving", []certv1.KeyUsage{certv1.UsageServerAuth}) +func GetCertificateSigningRequest(csrRequestName, commonName string, organization, san []string, signerName string) (*certv1.CertificateSigningRequest, []byte, error) { + + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, nil, fmt.Errorf("failed generating private key") + } + csrTemplate := x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: commonName, + Organization: organization, + }, + DNSNames: san, + } + + createCertificateRequest, err := x509.CreateCertificateRequest(rand.Reader, &csrTemplate, privateKey) + if err != nil { + return nil, nil, fmt.Errorf("failed to create CSR: %v", err) + } + + privateKeyPEM, err := encodePrivateKeyToPEM(privateKey) + if err != nil { + return nil, nil, fmt.Errorf("failed to encode private key: %v", err) + } + return (&certv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: csrRequestName, + Labels: map[string]string{ + "managed-by": "ceph-csi-operator", + }, + }, + Spec: certv1.CertificateSigningRequestSpec{ + Request: createCertificateRequest, + SignerName: signerName, + Usages: []certv1.KeyUsage{ + certv1.UsageDigitalSignature, + certv1.UsageKeyEncipherment, + certv1.UsageServerAuth, + }, + }, + }), privateKeyPEM, nil + +} + +func encodePrivateKeyToPEM(priv crypto.PrivateKey) ([]byte, error) { + privBytes, err := x509.MarshalECPrivateKey(priv.(*ecdsa.PrivateKey)) + if err != nil { + return nil, fmt.Errorf("failed to marshal private key: %v", err) + } + privPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: privBytes}) + return privPEM, nil +} diff --git a/internal/utils/csi.go b/internal/utils/csi.go index 56ad91ea..2a8e8e92 100644 --- a/internal/utils/csi.go +++ b/internal/utils/csi.go @@ -38,8 +38,10 @@ const ( CsiConfigMapConfigKey = "config.json" CsiConfigMapMappingKey = "cluster-mapping.json" - logsDirVolumeName = "logs-dir" - logRotateDirVolumeName = "log-rotate-dir" + logsDirVolumeName = "logs-dir" + logRotateDirVolumeName = "log-rotate-dir" + RBDNodePluginAddonsSidecarTlsSecretName = "tls-secret-node-plugin" + RBDCtrlPluginAddonsSidecarTlsSecretName = "tls-secret-ctrl-plugin" ) // Ceph CSI common volumes @@ -107,6 +109,38 @@ var OidcTokenVolume = corev1.Volume{ }, }, } +var TlsCertsRBDNodePluginVolume = corev1.Volume{ + Name: "certs", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: RBDNodePluginAddonsSidecarTlsSecretName, + }, + }, + }, + }, + }, + }, +} +var TlsCertsRBDCtrlPluginVolume = corev1.Volume{ + Name: "certs", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: RBDCtrlPluginAddonsSidecarTlsSecretName, + }, + }, + }, + }, + }, + }, +} var CsiConfigVolume = corev1.Volume{ Name: "ceph-csi-config", VolumeSource: corev1.VolumeSource{ @@ -271,6 +305,14 @@ var LogRotateDirVolumeMount = corev1.VolumeMount{ Name: logRotateDirVolumeName, MountPath: "/logrotate-config", } +var RBDNodePluginTLSCertVolumeMount = corev1.VolumeMount{ + Name: "certs", + MountPath: "/etc/tls", +} +var RBDCtrlPluginTLSCertVolumeMount = corev1.VolumeMount{ + Name: "certs", + MountPath: "/etc/tls", +} func PodsMountDirVolumeMount(kubletDirPath string) corev1.VolumeMount { return corev1.VolumeMount{