From 659c030e7718e8e097c556732fec104a365db7eb Mon Sep 17 00:00:00 2001 From: Andrew Bays Date: Mon, 13 May 2024 17:48:08 +0000 Subject: [PATCH] [OSPRH-6951] Auto-detect OSImage checksum name and type --- ...enstack.org_openstackprovisionservers.yaml | 26 +++ api/v1beta1/conditions.go | 18 ++ api/v1beta1/openstackprovisionserver_types.go | 19 +- api/v1beta1/zz_generated.deepcopy.go | 5 + ...enstack.org_openstackprovisionservers.yaml | 26 +++ config/default/manager_default_images.yaml | 3 +- config/rbac/role.yaml | 12 ++ containers/agent/checksum_discovery.go | 171 ++++++++++++++++++ containers/agent/main.go | 9 +- containers/agent/provision_ip_discovery.go | 82 ++++----- .../openstackbaremetalset_controller.go | 13 +- .../openstackprovisionserver_controller.go | 81 ++++++++- pkg/openstackbaremetalset/baremetalhost.go | 14 +- pkg/openstackprovisionserver/deployment.go | 5 +- pkg/openstackprovisionserver/initcontainer.go | 3 +- pkg/openstackprovisionserver/job.go | 96 ++++++++++ pkg/openstackprovisionserver/volumes.go | 32 ++-- .../config/httpd.conf | 4 +- .../openstackbaremetalset_controller_test.go | 3 + 19 files changed, 546 insertions(+), 76 deletions(-) create mode 100644 containers/agent/checksum_discovery.go create mode 100644 pkg/openstackprovisionserver/job.go diff --git a/api/bases/baremetal.openstack.org_openstackprovisionservers.yaml b/api/bases/baremetal.openstack.org_openstackprovisionservers.yaml index b46dd65..ac1ab24 100644 --- a/api/bases/baremetal.openstack.org_openstackprovisionservers.yaml +++ b/api/bases/baremetal.openstack.org_openstackprovisionservers.yaml @@ -74,10 +74,20 @@ spec: osImage: description: OSImage - OS qcow2 image (compressed as gz, or uncompressed) type: string + osImageDir: + default: /usr/local/apache2/htdocs + description: OSImageDir - Directory on the container which holds the + OS qcow2 image and checksum + type: string port: description: Port - The port on which the Apache server should listen format: int32 type: integer + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean resources: description: Resources - Compute Resources required by this provision server (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ @@ -132,6 +142,7 @@ spec: - apacheImageUrl - osContainerImageUrl - osImage + - osImageDir - port type: object status: @@ -186,9 +197,24 @@ spec: type: string description: Map of hashes to track e.g. job status type: object + localImageChecksumUrl: + description: URL of provisioning image checksum on underlying Apache + web server + type: string localImageUrl: description: URL of provisioning image on underlying Apache web server type: string + osImageChecksumFilename: + description: Filename of OSImage checksum + type: string + osImageChecksumType: + description: OSImage checksum type + enum: + - md5 + - sha256 + - sha512 + - auto + type: string provisionIp: description: IP of the provisioning interface on the node running the ProvisionServer pod diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index a7960f4..efe8248 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -33,6 +33,9 @@ const ( // OpenStackProvisionServerLocalImageURLReadyCondition Status=True condition which indicates if the OpenStackProvisionServer's LocalImageURL has been successfully acquired from the provisioning agent OpenStackProvisionServerLocalImageURLReadyCondition condition.Type = "OpenStackProvisionServerLocalImageURLReady" + // OpenStackProvisionServerChecksumReadyCondition Status=True condition which indicates if the OpenStackProvisionServer's OSImage Checksum has been successfully acquired from the provisioning agent + OpenStackProvisionServerChecksumReadyCondition condition.Type = "OpenStackProvisionServerChecksumReady" + // // OpenStackBaremetalSet conditions // @@ -87,6 +90,21 @@ const ( // OpenStackProvisionServerLocalImageURLReadyMessage OpenStackProvisionServerLocalImageURLReadyMessage = "OpenStackProvisionServerLocalImageURL generated" + // + // OpenStackProvisionServerChecksumReady condition messages + // + // OpenStackProvisionServerChecksumReadyInitMessage + OpenStackProvisionServerChecksumReadyInitMessage = "OpenStackProvisionServerChecksum not started" + + // OpenStackProvisionServerChecksumReadyErrorMessage + OpenStackProvisionServerChecksumReadyErrorMessage = "OpenStackProvisionServerChecksum error occured %s" + + // OpenStackProvisionServerChecksumReadyRunningMessage + OpenStackProvisionServerChecksumReadyRunningMessage = "OpenStackProvisionServerChecksum acquisition in progress" + + // OpenStackProvisionServerChecksumReadyMessage + OpenStackProvisionServerChecksumReadyMessage = "OpenStackProvisionServerChecksum acquired" + // // OpenStackBaremetalSetReady condition messages // diff --git a/api/v1beta1/openstackprovisionserver_types.go b/api/v1beta1/openstackprovisionserver_types.go index 57a995e..24ee943 100644 --- a/api/v1beta1/openstackprovisionserver_types.go +++ b/api/v1beta1/openstackprovisionserver_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta1 import ( + metal3v1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/util" corev1 "k8s.io/api/core/v1" @@ -27,11 +28,13 @@ import ( // +kubebuilder:validation:Enum=Managed;Unmanaged;Disabled type ProvisioningNetwork string -// ProvisioningNetwork modes const ( + // ProvisioningNetwork modes ProvisioningNetworkManaged ProvisioningNetwork = "Managed" ProvisioningNetworkUnmanaged ProvisioningNetwork = "Unmanaged" ProvisioningNetworkDisabled ProvisioningNetwork = "Disabled" + // Checksum job hash + ChecksumHash = "checksum" ) const ( @@ -54,6 +57,10 @@ type OpenStackProvisionServerSpec struct { Interface string `json:"interface,omitempty"` // OSImage - OS qcow2 image (compressed as gz, or uncompressed) OSImage string `json:"osImage"` + // +kubebuilder:validation:Required + // +kubebuilder:default=/usr/local/apache2/htdocs + // OSImageDir - Directory on the container which holds the OS qcow2 image and checksum + OSImageDir *string `json:"osImageDir"` // OSContainerImageURL - Container image URL for init with the OS qcow2 image (osImage) OSContainerImageURL string `json:"osContainerImageUrl"` // ApacheImageURL - Container image URL for the main container that serves the downloaded OS qcow2 image (osImage) @@ -67,6 +74,10 @@ type OpenStackProvisionServerSpec struct { // Resources - Compute Resources required by this provision server (Limits/Requests). // https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ Resources corev1.ResourceRequirements `json:"resources,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // PreserveJobs - do not delete jobs after they finished e.g. to check logs + PreserveJobs bool `json:"preserveJobs"` } // OpenStackProvisionServerStatus defines the observed state of OpenStackProvisionServer @@ -81,6 +92,12 @@ type OpenStackProvisionServerStatus struct { ProvisionIP string `json:"provisionIp,omitempty"` // URL of provisioning image on underlying Apache web server LocalImageURL string `json:"localImageUrl,omitempty"` + // Filename of OSImage checksum + OSImageChecksumFilename string `json:"osImageChecksumFilename,omitempty"` + // OSImage checksum type + OSImageChecksumType metal3v1.ChecksumType `json:"osImageChecksumType,omitempty"` + // URL of provisioning image checksum on underlying Apache web server + LocalImageChecksumURL string `json:"localImageChecksumUrl,omitempty"` } // IsReady - returns true if OpenStackProvisionServer is reconciled successfully diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index f359519..01666a9 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -460,6 +460,11 @@ func (in *OpenStackProvisionServerList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenStackProvisionServerSpec) DeepCopyInto(out *OpenStackProvisionServerSpec) { *out = *in + if in.OSImageDir != nil { + in, out := &in.OSImageDir, &out.OSImageDir + *out = new(string) + **out = **in + } if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector *out = make(map[string]string, len(*in)) diff --git a/config/crd/bases/baremetal.openstack.org_openstackprovisionservers.yaml b/config/crd/bases/baremetal.openstack.org_openstackprovisionservers.yaml index b46dd65..ac1ab24 100644 --- a/config/crd/bases/baremetal.openstack.org_openstackprovisionservers.yaml +++ b/config/crd/bases/baremetal.openstack.org_openstackprovisionservers.yaml @@ -74,10 +74,20 @@ spec: osImage: description: OSImage - OS qcow2 image (compressed as gz, or uncompressed) type: string + osImageDir: + default: /usr/local/apache2/htdocs + description: OSImageDir - Directory on the container which holds the + OS qcow2 image and checksum + type: string port: description: Port - The port on which the Apache server should listen format: int32 type: integer + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean resources: description: Resources - Compute Resources required by this provision server (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ @@ -132,6 +142,7 @@ spec: - apacheImageUrl - osContainerImageUrl - osImage + - osImageDir - port type: object status: @@ -186,9 +197,24 @@ spec: type: string description: Map of hashes to track e.g. job status type: object + localImageChecksumUrl: + description: URL of provisioning image checksum on underlying Apache + web server + type: string localImageUrl: description: URL of provisioning image on underlying Apache web server type: string + osImageChecksumFilename: + description: Filename of OSImage checksum + type: string + osImageChecksumType: + description: OSImage checksum type + enum: + - md5 + - sha256 + - sha512 + - auto + type: string provisionIp: description: IP of the provisioning interface on the node running the ProvisionServer pod diff --git a/config/default/manager_default_images.yaml b/config/default/manager_default_images.yaml index 7195906..0ca26ae 100644 --- a/config/default/manager_default_images.yaml +++ b/config/default/manager_default_images.yaml @@ -14,7 +14,8 @@ spec: - name: RELATED_IMAGE_OS_CONTAINER_IMAGE_URL_DEFAULT value: quay.io/podified-antelope-centos9/edpm-hardened-uefi:current-podified - name: RELATED_IMAGE_AGENT_IMAGE_URL_DEFAULT - value: quay.io/openstack-k8s-operators/openstack-baremetal-operator-agent:latest + # FIXME: REMOVE before merging + value: quay.io/andrewbays/openstack-baremetal-operator-agent:v0.0.1 - name: RELATED_IMAGE_APACHE_IMAGE_URL_DEFAULT value: registry.redhat.io/ubi9/httpd-24:latest - name: OS_IMAGE_DEFAULT diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f70af24..b49c6bf 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -100,6 +100,18 @@ rules: - list - patch - update +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/containers/agent/checksum_discovery.go b/containers/agent/checksum_discovery.go new file mode 100644 index 0000000..fd4b9e9 --- /dev/null +++ b/containers/agent/checksum_discovery.go @@ -0,0 +1,171 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "strings" + "time" + + "github.com/golang/glog" + metal3v1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" + "github.com/spf13/cobra" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +var ( + checksumStartCmd = &cobra.Command{ + Use: "checksum-discovery", + Short: "Start Checksum Discovery Agent", + Long: "", + Run: runChecksumStartCmd, + } + + checksumStartOpts struct { + kubeconfig string + osImageDir string + provServerName string + provServerNamespace string + } +) + +func init() { + rootCmd.AddCommand(checksumStartCmd) + checksumStartCmd.PersistentFlags().StringVar(&checksumStartOpts.osImageDir, "os-image-dir", "", "OS image directory on the associated host") + checksumStartCmd.PersistentFlags().StringVar(&checksumStartOpts.provServerName, "prov-server-name", "", "Provisioning server resource name") + checksumStartCmd.PersistentFlags().StringVar(&checksumStartOpts.provServerNamespace, "prov-server-namespace", "", "Provisioning server resource namespace") +} + +func runChecksumStartCmd(_ *cobra.Command, _ []string) { + var err error + err = flag.Set("logtostderr", "true") + if err != nil { + panic(err.Error()) + } + + flag.Parse() + + glog.V(0).Info("Starting ChecksumDiscoveryAgent") + + if checksumStartOpts.osImageDir == "" { + dir, ok := os.LookupEnv("OS_IMAGE_DIR") + if !ok || dir == "" { + glog.Fatalf("os-image-dir is required") + } + checksumStartOpts.osImageDir = dir + } + + if checksumStartOpts.provServerName == "" { + name, ok := os.LookupEnv("PROV_SERVER_NAME") + if !ok || name == "" { + glog.Fatalf("prov-server-name is required") + } + checksumStartOpts.provServerName = name + } + + if checksumStartOpts.provServerNamespace == "" { + name, ok := os.LookupEnv("PROV_SERVER_NAMESPACE") + if !ok || name == "" { + glog.Fatalf("prov-server-namespace is required") + } + checksumStartOpts.provServerNamespace = name + } + + var config *rest.Config + kubeconfig := os.Getenv("KUBECONFIG") + if kubeconfig != "" { + config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) + } else { + // creates the in-cluster config + config, err = rest.InClusterConfig() + } + + if err != nil { + panic(err.Error()) + } + + dClient := dynamic.NewForConfigOrDie(config) + + provServerClient := dClient.Resource(openstackProvisionServerGVR) + + checksumFileName := "" + var checksumType metal3v1.ChecksumType + + // First get the checksum data + dir, err := os.Open(checksumStartOpts.osImageDir) + + if err != nil { + panic(err.Error()) + } + + items, err := dir.Readdirnames(0) + _ = dir.Close() + + if err != nil { + panic(err.Error()) + } + + for _, item := range items { + // Crude mechanism for detecting both the checksum file and its type + if strings.Contains(item, "md5") { + checksumFileName = item + checksumType = metal3v1.MD5 + break + } else if strings.Contains(item, "sha256") { + checksumFileName = item + checksumType = metal3v1.SHA256 + break + } else if strings.Contains(item, "sha512") { + checksumFileName = item + checksumType = metal3v1.SHA512 + break + } + } + + if checksumFileName == "" { + panic(fmt.Errorf("could not find any OSImage checksum file in %s", checksumStartOpts.osImageDir)) + } + + // Try to update status with checksum data until it succeeds, as it's possible to hit "object has been modified" k8s error here + for { + unstructured, err := provServerClient.Namespace(checksumStartOpts.provServerNamespace).Get(context.Background(), checksumStartOpts.provServerName, metav1.GetOptions{}, "/status") + + if k8s_errors.IsNotFound(err) { + // Deleted somehow, so just break + break + } + + if err != nil { + panic(err.Error()) + } + + if unstructured.Object["status"] == nil { + unstructured.Object["status"] = map[string]interface{}{} + } + + status := unstructured.Object["status"].(map[string]interface{}) + + status["osImageChecksumFilename"] = checksumFileName + status["osImageChecksumType"] = checksumType + + unstructured.Object["status"] = status + + _, err = provServerClient.Namespace(checksumStartOpts.provServerNamespace).UpdateStatus(context.Background(), unstructured, metav1.UpdateOptions{}) + + if err != nil { + glog.V(0).Infof("Error updating OpenStackProvisionServer %s (namespace %s) \"osImageChecksumFilename\" and \"osImageChecksumType\" status: %s\n", checksumStartOpts.provServerName, checksumStartOpts.provServerNamespace, err) + } else { + glog.V(0).Infof("Updated OpenStackProvisionServer %s (namespace %s) with status \"osImageChecksumFilename\": %s and \"osImageChecksumType\": %s\n", checksumStartOpts.provServerName, checksumStartOpts.provServerNamespace, checksumFileName, checksumType) + break + } + + time.Sleep(time.Second * 1) + } + + glog.V(0).Info("Shutting down ChecksumDiscoveryAgent") +} diff --git a/containers/agent/main.go b/containers/agent/main.go index c92efe0..d7ca210 100644 --- a/containers/agent/main.go +++ b/containers/agent/main.go @@ -5,6 +5,7 @@ import ( "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/runtime/schema" ) const ( @@ -17,6 +18,12 @@ var ( Short: "Run OpenStack Agent", Long: "Runs the OpenStack Baremetal Operator Agent", } + + openstackProvisionServerGVR = schema.GroupVersionResource{ + Group: "baremetal.openstack.org", + Version: "v1beta1", + Resource: "openstackprovisionservers", + } ) func init() { @@ -25,6 +32,6 @@ func init() { func main() { if err := rootCmd.Execute(); err != nil { - glog.Exitf("Error executing mcd: %v", err) + glog.Exitf("Error executing cmd: %v", err) } } diff --git a/containers/agent/provision_ip_discovery.go b/containers/agent/provision_ip_discovery.go index 7a45598..d09f5c7 100644 --- a/containers/agent/provision_ip_discovery.go +++ b/containers/agent/provision_ip_discovery.go @@ -11,42 +11,35 @@ import ( "github.com/spf13/cobra" k8s_errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) var ( - startCmd = &cobra.Command{ + provisionIPStartCmd = &cobra.Command{ Use: "provision-ip-discovery", Short: "Start Provision IP Discovery Agent", Long: "", - Run: runStartCmd, + Run: runProvisionIPStartCmd, } - startOpts struct { + provisionIPStartOpts struct { kubeconfig string provIntf string provServerName string provServerNamespace string } - - openstackProvisionServerGVR = schema.GroupVersionResource{ - Group: "baremetal.openstack.org", - Version: "v1beta1", - Resource: "openstackprovisionservers", - } ) func init() { - rootCmd.AddCommand(startCmd) - startCmd.PersistentFlags().StringVar(&startOpts.provIntf, "prov-intf", "", "Provisioning interface name on the associated host") - startCmd.PersistentFlags().StringVar(&startOpts.provServerName, "prov-server-name", "", "Provisioning server resource name") - startCmd.PersistentFlags().StringVar(&startOpts.provServerNamespace, "prov-server-namespace", "", "Provisioning server resource namespace") + rootCmd.AddCommand(provisionIPStartCmd) + provisionIPStartCmd.PersistentFlags().StringVar(&provisionIPStartOpts.provIntf, "prov-intf", "", "Provisioning interface name on the associated host") + provisionIPStartCmd.PersistentFlags().StringVar(&provisionIPStartOpts.provServerName, "prov-server-name", "", "Provisioning server resource name") + provisionIPStartCmd.PersistentFlags().StringVar(&provisionIPStartOpts.provServerNamespace, "prov-server-namespace", "", "Provisioning server resource namespace") } -func runStartCmd(_ *cobra.Command, _ []string) { +func runProvisionIPStartCmd(_ *cobra.Command, _ []string) { var err error err = flag.Set("logtostderr", "true") if err != nil { @@ -57,28 +50,28 @@ func runStartCmd(_ *cobra.Command, _ []string) { glog.V(0).Info("Starting ProvisionIpDiscoveryAgent") - if startOpts.provIntf == "" { + if provisionIPStartOpts.provIntf == "" { name, ok := os.LookupEnv("PROV_INTF") if !ok || name == "" { glog.Fatalf("prov-intf is required") } - startOpts.provIntf = name + provisionIPStartOpts.provIntf = name } - if startOpts.provServerName == "" { + if provisionIPStartOpts.provServerName == "" { name, ok := os.LookupEnv("PROV_SERVER_NAME") if !ok || name == "" { glog.Fatalf("prov-server-name is required") } - startOpts.provServerName = name + provisionIPStartOpts.provServerName = name } - if startOpts.provServerNamespace == "" { + if provisionIPStartOpts.provServerNamespace == "" { name, ok := os.LookupEnv("PROV_SERVER_NAMESPACE") if !ok || name == "" { glog.Fatalf("prov-server-namespace is required") } - startOpts.provServerNamespace = name + provisionIPStartOpts.provServerNamespace = name } var config *rest.Config @@ -100,18 +93,9 @@ func runStartCmd(_ *cobra.Command, _ []string) { ip := "" + // Get provision interface IP and update the status, and then sleep 5 seconds + // and check again over and over (because the IP address could change) for { - unstructured, err := provServerClient.Namespace(startOpts.provServerNamespace).Get(context.Background(), startOpts.provServerName, metav1.GetOptions{}) - - if k8s_errors.IsNotFound(err) { - // Deleted somehow, so just break - break - } - - if err != nil { - panic(err.Error()) - } - ifaces, err := net.Interfaces() if err != nil { @@ -121,7 +105,7 @@ func runStartCmd(_ *cobra.Command, _ []string) { curIP := "" for _, iface := range ifaces { - if iface.Name == startOpts.provIntf { + if iface.Name == provisionIPStartOpts.provIntf { addrs, err := iface.Addrs() if err != nil { @@ -132,7 +116,7 @@ func runStartCmd(_ *cobra.Command, _ []string) { ipObj, _, err := net.ParseCIDR(addr.String()) if err != nil || ipObj == nil { - glog.V(0).Infof("WARNING: Cannot parse IP address for OpenStackProvisionServer %s (namespace %s) on interface %s!\n", startOpts.provServerName, startOpts.provServerName, startOpts.provIntf) + glog.V(0).Infof("WARNING: Cannot parse IP address for OpenStackProvisionServer %s (namespace %s) on interface %s!\n", provisionIPStartOpts.provServerName, provisionIPStartOpts.provServerName, provisionIPStartOpts.provIntf) if err != nil { glog.V(0).Infof("ERROR: %s", err.Error()) } @@ -143,27 +127,43 @@ func runStartCmd(_ *cobra.Command, _ []string) { curIP = ipObj.String() break } - glog.V(0).Infof("INFO: Ignoring IPv6 address (%s) for OpenStackProvisionServer %s (namespace %s) on interface %s!\n", addr, startOpts.provServerName, startOpts.provServerName, startOpts.provIntf) + glog.V(0).Infof("INFO: Ignoring IPv6 address (%s) for OpenStackProvisionServer %s (namespace %s) on interface %s!\n", addr, provisionIPStartOpts.provServerName, provisionIPStartOpts.provServerName, provisionIPStartOpts.provIntf) } break } } if curIP == "" { - glog.V(0).Infof("WARNING: Unable to find provisioning IP for OpenStackProvisionServer %s (namespace %s) on interface %s!\n", startOpts.provServerName, startOpts.provServerName, startOpts.provIntf) + glog.V(0).Infof("ERROR: Unable to find provisioning IP for OpenStackProvisionServer %s (namespace %s) on interface %s!\n", provisionIPStartOpts.provServerName, provisionIPStartOpts.provServerName, provisionIPStartOpts.provIntf) } else if ip != curIP { + unstructured, err := provServerClient.Namespace(provisionIPStartOpts.provServerNamespace).Get(context.Background(), provisionIPStartOpts.provServerName, metav1.GetOptions{}, "/status") - unstructured.Object["status"] = map[string]interface{}{ - "provisionIp": curIP, + if k8s_errors.IsNotFound(err) { + // Deleted somehow, so just break + break } - _, err = provServerClient.Namespace(startOpts.provServerNamespace).UpdateStatus(context.Background(), unstructured, metav1.UpdateOptions{}) + if err != nil { + panic(err.Error()) + } + + if unstructured.Object["status"] == nil { + unstructured.Object["status"] = map[string]interface{}{} + } + + status := unstructured.Object["status"].(map[string]interface{}) + + status["provisionIp"] = curIP + + unstructured.Object["status"] = status + + _, err = provServerClient.Namespace(provisionIPStartOpts.provServerNamespace).UpdateStatus(context.Background(), unstructured, metav1.UpdateOptions{}) if err != nil { - glog.V(0).Infof("Error updating OpenStackProvisionServer %s (namespace %s) \"provisionIp\" status: %s\n", startOpts.provServerName, startOpts.provServerNamespace, err) + glog.V(0).Infof("Error updating OpenStackProvisionServer %s (namespace %s) \"provisionIp\" status: %s\n", provisionIPStartOpts.provServerName, provisionIPStartOpts.provServerNamespace, err) } else { ip = curIP - glog.V(0).Infof("Updated OpenStackProvisionServer %s (namespace %s) with status \"provisionIp\": %s\n", startOpts.provServerName, startOpts.provServerNamespace, ip) + glog.V(0).Infof("Updated OpenStackProvisionServer %s (namespace %s) with status \"provisionIp\": %s\n", provisionIPStartOpts.provServerName, provisionIPStartOpts.provServerNamespace, ip) } } diff --git a/controllers/openstackbaremetalset_controller.go b/controllers/openstackbaremetalset_controller.go index 61401f5..44c25bd 100644 --- a/controllers/openstackbaremetalset_controller.go +++ b/controllers/openstackbaremetalset_controller.go @@ -370,6 +370,17 @@ func (r *OpenStackBaremetalSetReconciler) reconcileNormal(ctx context.Context, i l.Info("OpenStackProvisionServer LocalImageURL not yet available", "OpenStackProvisionServer", provisionServer.Name) return ctrl.Result{RequeueAfter: time.Duration(30) * time.Second}, nil } + + if provisionServer.Status.LocalImageChecksumURL == "" { + instance.Status.Conditions.Set(condition.FalseCondition( + baremetalv1.OpenStackBaremetalSetProvServerReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + baremetalv1.OpenStackBaremetalSetProvServerReadyRunningMessage)) + l.Info("OpenStackProvisionServer LocalImageChecksumURL not yet available", "OpenStackProvisionServer", provisionServer.Name) + return ctrl.Result{RequeueAfter: time.Duration(5) * time.Second}, nil + } + instance.Status.Conditions.MarkTrue(baremetalv1.OpenStackBaremetalSetProvServerReadyCondition, baremetalv1.OpenStackBaremetalSetProvServerReadyMessage) // handle provision server - end @@ -628,7 +639,7 @@ func (r *OpenStackBaremetalSetReconciler) ensureBaremetalHosts( bmhName, desiredHostName, instance.Spec.BaremetalHosts[desiredHostName].CtlPlaneIP, // ctlPlaneIP - provisionServer.Status.LocalImageURL, + provisionServer, sshSecret, passwordSecret, envVars, diff --git a/controllers/openstackprovisionserver_controller.go b/controllers/openstackprovisionserver_controller.go index 68ec5c6..ba77805 100644 --- a/controllers/openstackprovisionserver_controller.go +++ b/controllers/openstackprovisionserver_controller.go @@ -47,6 +47,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/deployment" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/job" "github.com/openstack-k8s-operators/lib-common/modules/common/labels" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" "github.com/openstack-k8s-operators/lib-common/modules/common/util" @@ -87,6 +88,7 @@ type OpenStackProvisionServerReconciler struct { // +kubebuilder:rbac:groups=core,resources=volumes,verbs=get;list;create;update;delete;watch;patch // +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;update;watch;patch // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;update;watch;patch +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=metal3.io,resources=provisionings,verbs=get;list;watch // +kubebuilder:rbac:groups=metal3.io,resources=baremetalhosts,verbs=get;list;update;patch;watch @@ -171,6 +173,11 @@ func (r *OpenStackProvisionServerReconciler) Reconcile(ctx context.Context, req condition.InitReason, baremetalv1.OpenStackProvisionServerLocalImageURLReadyInitMessage, ), + condition.UnknownCondition( + baremetalv1.OpenStackProvisionServerChecksumReadyCondition, + condition.InitReason, + baremetalv1.OpenStackProvisionServerChecksumReadyInitMessage, + ), // service account, role, rolebinding conditions condition.UnknownCondition( @@ -431,11 +438,14 @@ func (r *OpenStackProvisionServerReconciler) reconcileNormal(ctx context.Context } // create Deployment - end - instance.Status.LocalImageURL, err = r.getLocalImageURL(ctx, helper, instance) + instance.Status.LocalImageURL, err = r.getLocalImageURL(ctx, helper, instance, instance.Spec.OSImage) if err != nil { instance.Status.Conditions.MarkFalse( - baremetalv1.OpenStackProvisionServerLocalImageURLReadyCondition, condition.ErrorReason, condition.SeverityError, - baremetalv1.OpenStackBaremetalSetBmhProvisioningReadyErrorMessage, err.Error()) + baremetalv1.OpenStackProvisionServerLocalImageURLReadyCondition, + condition.ErrorReason, + condition.SeverityError, + baremetalv1.OpenStackProvisionServerLocalImageURLReadyErrorMessage, + err.Error()) return ctrl.Result{}, err } @@ -456,6 +466,65 @@ func (r *OpenStackProvisionServerReconciler) reconcileNormal(ctx context.Context } // check ProvisionIp/LocalImageURL - end + checksumHash := instance.Status.Hash[baremetalv1.ChecksumHash] + jobDef := openstackprovisionserver.ChecksumJob(instance, serviceLabels, map[string]string{}) + checksumJob := job.NewJob( + jobDef, + baremetalv1.ChecksumHash, + instance.Spec.PreserveJobs, + 5*time.Second, + checksumHash, + ) + ctrlResult, err = checksumJob.DoJob( + ctx, + helper, + ) + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + baremetalv1.OpenStackProvisionServerChecksumReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + baremetalv1.OpenStackProvisionServerChecksumReadyRunningMessage)) + return ctrlResult, nil + } + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + baremetalv1.OpenStackProvisionServerChecksumReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + baremetalv1.OpenStackProvisionServerChecksumReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if checksumJob.HasChanged() { + instance.Status.Hash[baremetalv1.ChecksumHash] = checksumJob.GetHash() + r.Log.Info(fmt.Sprintf("Job %s hash added - %s", jobDef.Name, instance.Status.Hash[baremetalv1.ChecksumHash])) + } + + instance.Status.LocalImageChecksumURL, err = r.getLocalImageURL(ctx, helper, instance, instance.Status.OSImageChecksumFilename) + if err != nil { + instance.Status.Conditions.MarkFalse( + baremetalv1.OpenStackProvisionServerChecksumReadyCondition, + condition.ErrorReason, + condition.SeverityError, + baremetalv1.OpenStackProvisionServerChecksumReadyErrorMessage, + err.Error()) + return ctrl.Result{}, err + } + + if instance.Status.LocalImageChecksumURL != "" { + instance.Status.Conditions.MarkTrue(baremetalv1.OpenStackProvisionServerChecksumReadyCondition, baremetalv1.OpenStackProvisionServerChecksumReadyMessage) + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + baremetalv1.OpenStackProvisionServerChecksumReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + baremetalv1.OpenStackProvisionServerChecksumReadyRunningMessage)) + + return ctrl.Result{RequeueAfter: time.Duration(5) * time.Second}, nil + } + // checksum job - end + // We reached the end of the Reconcile, update the Ready condition based on // the sub conditions if instance.Status.Conditions.AllSubConditionIsTrue() { @@ -484,6 +553,7 @@ func (r *OpenStackProvisionServerReconciler) generateServiceConfigMaps( templateParameters := make(map[string]interface{}) templateParameters["Port"] = strconv.FormatInt(int64(instance.Spec.Port), 10) + templateParameters["DocumentRoot"] = instance.Spec.OSImageDir cms := []util.Template{ // Apache server config @@ -599,8 +669,7 @@ func (r *OpenStackProvisionServerReconciler) getProvisioningInterface( } func (r *OpenStackProvisionServerReconciler) getLocalImageURL( - ctx context.Context, helper *helper.Helper, instance *baremetalv1.OpenStackProvisionServer) (string, error) { - baseFilename := instance.Spec.OSImage + ctx context.Context, helper *helper.Helper, instance *baremetalv1.OpenStackProvisionServer, filename string) (string, error) { host := instance.Status.ProvisionIP if host == "" { serviceLabels := labels.GetLabels(instance, openstackprovisionserver.AppLabel, map[string]string{ @@ -619,5 +688,5 @@ func (r *OpenStackProvisionServerReconciler) getLocalImageURL( if k8snet.IsIPv6(net.ParseIP(host)) { host = fmt.Sprintf("[%s]", host) } - return fmt.Sprintf("http://%s:%d/%s", host, instance.Spec.Port, baseFilename), nil + return fmt.Sprintf("http://%s:%d/%s", host, instance.Spec.Port, filename), nil } diff --git a/pkg/openstackbaremetalset/baremetalhost.go b/pkg/openstackbaremetalset/baremetalhost.go index 27f8ffe..4241d74 100644 --- a/pkg/openstackbaremetalset/baremetalhost.go +++ b/pkg/openstackbaremetalset/baremetalhost.go @@ -28,7 +28,7 @@ func BaremetalHostProvision( bmh string, hostName string, ctlPlaneIP string, - localImageURL string, + provServer *baremetalv1.OpenStackProvisionServer, sshSecret *corev1.Secret, passwordSecret *corev1.Secret, envVars *map[string]env.Setter, @@ -216,9 +216,9 @@ func BaremetalHostProvision( // if foundBaremetalHost.Status.Provisioning.State != metal3v1.StateProvisioned { foundBaremetalHost.Spec.Image = &metal3v1.Image{ - URL: localImageURL, - Checksum: fmt.Sprintf("%s.sha256", localImageURL), - ChecksumType: metal3v1.SHA256, + URL: provServer.Status.LocalImageURL, + Checksum: provServer.Status.LocalImageChecksumURL, + ChecksumType: provServer.Status.OSImageChecksumType, } } @@ -229,9 +229,9 @@ func BaremetalHostProvision( foundBaremetalHost.Spec.Online = true foundBaremetalHost.Spec.ConsumerRef = &corev1.ObjectReference{Name: instance.Name, Kind: instance.Kind, Namespace: instance.Namespace} foundBaremetalHost.Spec.Image = &metal3v1.Image{ - URL: localImageURL, - Checksum: fmt.Sprintf("%s.sha256", localImageURL), - ChecksumType: metal3v1.SHA256, + URL: provServer.Status.LocalImageURL, + Checksum: provServer.Status.LocalImageChecksumURL, + ChecksumType: provServer.Status.OSImageChecksumType, } foundBaremetalHost.Spec.UserData = userDataSecret foundBaremetalHost.Spec.NetworkData = networkDataSecret diff --git a/pkg/openstackprovisionserver/deployment.go b/pkg/openstackprovisionserver/deployment.go index b16c9d1..68a2813 100644 --- a/pkg/openstackprovisionserver/deployment.go +++ b/pkg/openstackprovisionserver/deployment.go @@ -88,7 +88,7 @@ func Deployment( }, Args: args, Image: instance.Spec.ApacheImageURL, - VolumeMounts: getVolumeMounts(), + VolumeMounts: getVolumeMounts(instance), Resources: instance.Spec.Resources, StartupProbe: startupProbe, ReadinessProbe: readinessProbe, @@ -183,8 +183,9 @@ func Deployment( } initContainerDetails := InitContainerDetails{ + OsImageDir: *instance.Spec.OSImageDir, ContainerImage: instance.Spec.OSContainerImageURL, - VolumeMounts: getInitVolumeMounts(), + VolumeMounts: getInitVolumeMounts(instance), } deployment.Spec.Template.Spec.InitContainers = InitContainer(initContainerDetails) diff --git a/pkg/openstackprovisionserver/initcontainer.go b/pkg/openstackprovisionserver/initcontainer.go index dad8653..6775337 100644 --- a/pkg/openstackprovisionserver/initcontainer.go +++ b/pkg/openstackprovisionserver/initcontainer.go @@ -8,6 +8,7 @@ import ( // InitContainerDetails information type InitContainerDetails struct { ContainerImage string + OsImageDir string Privileged bool VolumeMounts []corev1.VolumeMount } @@ -17,7 +18,7 @@ func InitContainer(init InitContainerDetails) []corev1.Container { envs := []corev1.EnvVar{ { Name: "DEST_DIR", - Value: "/usr/local/apache2/htdocs", + Value: init.OsImageDir, }, } envs = env.MergeEnvs(envs, map[string]env.Setter{}) diff --git a/pkg/openstackprovisionserver/job.go b/pkg/openstackprovisionserver/job.go new file mode 100644 index 0000000..ba83e23 --- /dev/null +++ b/pkg/openstackprovisionserver/job.go @@ -0,0 +1,96 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openstackprovisionserver + +import ( + "fmt" + + baremetalv1 "github.com/openstack-k8s-operators/openstack-baremetal-operator/api/v1beta1" + + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // ChecksumCommand - + ChecksumCommand = "/openstack-baremetal-agent checksum-discovery" +) + +// ChecksumJob func +func ChecksumJob( + instance *baremetalv1.OpenStackProvisionServer, + labels map[string]string, + annotations map[string]string, +) *batchv1.Job { + args := []string{"-c", ChecksumCommand} + + envVars := map[string]env.Setter{} + envVars["OS_IMAGE_DIR"] = env.SetValue(*instance.Spec.OSImageDir) + envVars["PROV_SERVER_NAME"] = env.SetValue(instance.Name) + envVars["PROV_SERVER_NAMESPACE"] = env.SetValue(instance.Namespace) + + // We actually use init volumes and mounts for this job + volumes := getInitVolumes() + volumeMounts := getInitVolumeMounts(instance) + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-checksum-discovery", instance.Name), + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: instance.RbacResourceName(), + Containers: []corev1.Container{ + { + Name: fmt.Sprintf("%s-checksum-discovery", instance.Name), + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.AgentImageURL, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + if instance.Spec.NodeSelector != nil && len(instance.Spec.NodeSelector) > 0 { + job.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector + } + + initContainerDetails := InitContainerDetails{ + OsImageDir: *instance.Spec.OSImageDir, + ContainerImage: instance.Spec.OSContainerImageURL, + VolumeMounts: getInitVolumeMounts(instance), + } + job.Spec.Template.Spec.InitContainers = InitContainer(initContainerDetails) + + return job +} diff --git a/pkg/openstackprovisionserver/volumes.go b/pkg/openstackprovisionserver/volumes.go index d915153..090ff17 100644 --- a/pkg/openstackprovisionserver/volumes.go +++ b/pkg/openstackprovisionserver/volumes.go @@ -16,11 +16,12 @@ package openstackprovisionserver import ( "fmt" + baremetalv1 "github.com/openstack-k8s-operators/openstack-baremetal-operator/api/v1beta1" corev1 "k8s.io/api/core/v1" ) // getVolumes - general provisioning service volumes -func getVolumes(name string) []corev1.Volume { +func getInitVolumes() []corev1.Volume { return []corev1.Volume{ { Name: "image-data", @@ -28,35 +29,40 @@ func getVolumes(name string) []corev1.Volume { EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, - { - Name: "httpd-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: fmt.Sprintf("%s-httpd-config", name), - }, + } +} + +// getVolumes - general provisioning service volumes +func getVolumes(name string) []corev1.Volume { + return append(getInitVolumes(), corev1.Volume{ + Name: "httpd-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: fmt.Sprintf("%s-httpd-config", name), }, }, }, - } + }, + ) } // getInitVolumeMounts - general init task VolumeMounts -func getInitVolumeMounts() []corev1.VolumeMount { +func getInitVolumeMounts(instance *baremetalv1.OpenStackProvisionServer) []corev1.VolumeMount { return []corev1.VolumeMount{ { Name: "image-data", - MountPath: "/usr/local/apache2/htdocs", + MountPath: *instance.Spec.OSImageDir, }, } } // getVolumeMounts - general VolumeMounts -func getVolumeMounts() []corev1.VolumeMount { +func getVolumeMounts(instance *baremetalv1.OpenStackProvisionServer) []corev1.VolumeMount { return []corev1.VolumeMount{ { Name: "image-data", - MountPath: "/usr/local/apache2/htdocs", + MountPath: *instance.Spec.OSImageDir, }, { Name: "httpd-config", diff --git a/templates/openstackprovisionserver/config/httpd.conf b/templates/openstackprovisionserver/config/httpd.conf index 1086c0c..e1a95e8 100644 --- a/templates/openstackprovisionserver/config/httpd.conf +++ b/templates/openstackprovisionserver/config/httpd.conf @@ -116,8 +116,8 @@ ServerAdmin root@localhost # documents. By default, all requests are taken from this directory, but # symbolic links and aliases may be used to point to other locations. # -DocumentRoot "/usr/local/apache2/htdocs" - +DocumentRoot "{{ .DocumentRoot }}" + # # Possible values for the Options directive are "None", "All", # or any combination of: diff --git a/tests/functional/openstackbaremetalset_controller_test.go b/tests/functional/openstackbaremetalset_controller_test.go index 6f66115..bbb702b 100644 --- a/tests/functional/openstackbaremetalset_controller_test.go +++ b/tests/functional/openstackbaremetalset_controller_test.go @@ -172,11 +172,14 @@ var _ = Describe("BaremetalSet Test", func() { DeferCleanup(th.DeleteInstance, CreateBaremetalSet(baremetalSetName, DefaultBaremetalSetSpec(bmhName, true))) }) It("Prov Server should have the Spec fields initialized", func() { + osImageDir := "/usr/local/apache2/htdocs" + provServer := GetProvisionServer(baremetalSetName) spec := baremetalv1.OpenStackProvisionServerSpec{ Port: 6190, Interface: "eth1", OSImage: "edpm-hardened-uefi.qcow2", + OSImageDir: &osImageDir, OSContainerImageURL: "quay.io/podified-antelope-centos9/edpm-hardened-uefi@latest", ApacheImageURL: "registry.redhat.io/rhel8/httpd-24@latest", AgentImageURL: "quay.io/openstack-k8s-operators/openstack-baremetal-operator-agent@latest",