From 0cc7bfe37f46578ac1b70f10654329815cab79bf Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Fri, 11 Aug 2023 11:35:04 +0200 Subject: [PATCH] Update horizon to use service override Removes creation of routes.Those get done in the openstack-operator. Via service overrides the service can be customized. The service operator adds annotation to the service which needs to be exposed as a route. Jira: OSP-26690 Depends-On: openstack-k8s-operators/lib-common#332 --- README.md | 4 - api/bases/horizon.openstack.org_horizons.yaml | 200 +++++++++++++++--- api/go.mod | 4 + api/go.sum | 4 +- api/v1beta1/horizon_types.go | 42 ++-- api/v1beta1/zz_generated.deepcopy.go | 40 ++-- .../bases/horizon.openstack.org_horizons.yaml | 200 +++++++++++++++--- config/rbac/role.yaml | 12 -- controllers/horizon_controller.go | 129 ++++++++--- go.mod | 6 +- go.sum | 8 +- main.go | 7 +- tests/functional/horizon_controller_test.go | 4 - tests/functional/suite_test.go | 7 - 14 files changed, 497 insertions(+), 170 deletions(-) diff --git a/README.md b/README.md index b67e629f..01db016c 100644 --- a/README.md +++ b/README.md @@ -79,8 +79,6 @@ template: preserveJobs: false replicas: 1 resources: {} - route: - routeName: horizon secret: osp-secret ``` @@ -110,8 +108,6 @@ template: preserveJobs: false replicas: 1 resources: {} - route: - routeName: horizon secret: osp-secret memcachedInstance: my-custom-memcached #<<-- Custom memcached instance supplied here. ``` diff --git a/api/bases/horizon.openstack.org_horizons.yaml b/api/bases/horizon.openstack.org_horizons.yaml index d7bb6b54..afde2584 100644 --- a/api/bases/horizon.openstack.org_horizons.yaml +++ b/api/bases/horizon.openstack.org_horizons.yaml @@ -83,6 +83,180 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: RoutedOverrideSpec - a routed service override + configuration for the Service created to serve traffic to + the cluster. Allows for the manifest of the created Service + to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: EmbeddedLabelsAnnotations is an embedded subset + of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value + map stored with a resource that may be set by external + tools to store and retrieve arbitrary metadata. They + are not queryable and should be preserved when modifying + objects. More info: http://kubernetes.io/docs/user-guide/annotations' + type: object + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can + be used to organize and categorize (scope and select) + objects. May match selectors of replication controllers + and services. More info: http://kubernetes.io/docs/user-guide/labels' + type: object + type: object + spec: + description: OverrideServiceSpec is a subset of the fields + included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, + ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: externalName is the external reference + that discovery mechanisms will return as an alias + for this service (e.g. a DNS CNAME record). No proxying + will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires + `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: externalTrafficPolicy describes how nodes + distribute service traffic they receive on one of + the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", + the proxy will configure the service in a way that + assumes that external load balancers will take care + of balancing the service traffic between nodes, and + so each node will deliver traffic only to the node-local + endpoints of the service, without masquerading the + client source IP. (Traffic mistakenly sent to a node + with no endpoints will be dropped.) The default value, + "Cluster", uses the standard behavior of routing to + all endpoints evenly (possibly modified by topology + and other features). Note that traffic sent to an + External IP or LoadBalancer IP from within the cluster + will always get "Cluster" semantics, but clients sending + to a NodePort from within the cluster may need to + take traffic policy into account when picking a node. + type: string + internalTrafficPolicy: + description: InternalTrafficPolicy describes how nodes + distribute service traffic they receive on the ClusterIP. + If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the + same node as the pod, dropping the traffic if there + are no local endpoints. The default value, "Cluster", + uses the standard behavior of routing to all endpoints + evenly (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: IPFamilyPolicy represents the dual-stack-ness + requested or required by this Service. If there is + no value provided, then this field will be set to + SingleStack. Services can be "SingleStack" (a single + IP family), "PreferDualStack" (two IP families on + dual-stack configured clusters or a single IP family + on single-stack clusters), or "RequireDualStack" (two + IP families on dual-stack configured clusters, otherwise + fail). The ipFamilies and clusterIPs fields depend + on the value of this field. This field will be wiped + when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: loadBalancerClass is the class of the load + balancer implementation this Service belongs to. If + specified, the value of this field must be a label-style + identifier, with an optional prefix, e.g. "internal-vip" + or "example.com/internal-vip". Unprefixed names are + reserved for end-users. This field can only be set + when the Service type is 'LoadBalancer'. If not set, + the default load balancer implementation is used, + today this is typically done through the cloud provider + integration, but should apply for any default implementation. + If set, it is assumed that a load balancer implementation + is watching for Services with a matching class. Any + default load balancer implementation (e.g. cloud providers) + should ignore Services that set this field. This field + can only be set when creating or updating a Service + to type 'LoadBalancer'. Once set, it can not be changed. + This field will be wiped when a service is updated + to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified + client IPs. This field will be ignored if the cloud-provider + does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/' + items: + type: string + type: array + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to + maintain session affinity. Enable client IP based + session affinity. Must be ClientIP or None. Defaults + to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The + value must be >0 && <=86400(for 1 day) if + ServiceAffinity == "ClientIP". Default value + is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, + ClusterIP, NodePort, and LoadBalancer. "ClusterIP" + allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector + or if that is not specified, by manual construction + of an Endpoints object or EndpointSlice objects. If + clusterIP is "None", no virtual IP is allocated and + the endpoints are published as a set of endpoints + rather than a virtual IP. "NodePort" builds on ClusterIP + and allocates a port on every node which routes to + the same endpoints as the clusterIP. "LoadBalancer" + builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to + the same endpoints as the clusterIP. "ExternalName" + aliases this service to the specified externalName. + Several other fields do not apply to ExternalName + services. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + type: object + description: Override configuration for the Service created to + serve traffic to the cluster. The key must be the endpoint type + (public, internal) + type: object + type: object preserveJobs: default: false description: PreserveJobs - do not delete jobs after they finished @@ -143,32 +317,6 @@ spec: to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object - route: - description: HorizonRoute holds all of the necessary options for configuring - the Horizon Route object. This can be used to configure TLS TODO(bshephar) - Implement everything about this. It's just a placeholder at the - moment. - properties: - routeLocation: - description: TODO(bshephar) We need to implement TLS handling - here to secure the route - type: string - routeName: - default: horizon - type: string - routeTLSCA: - description: TODO(bshephar) We need to implement TLS handling - here to secure the route - type: string - routeTLSEnabled: - description: TODO(bshephar) We need to implement TLS handling - here to secure the route - type: string - routeTLSKey: - description: TODO(bshephar) We need to implement TLS handling - here to secure the route - type: string - type: object secret: description: Secret containing OpenStack password information for Horizon Secret Key diff --git a/api/go.mod b/api/go.mod index be626a25..24cd85d4 100644 --- a/api/go.mod +++ b/api/go.mod @@ -66,3 +66,7 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) + +replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/stuggi/lib-common/modules/common v0.0.0-20230911170028-26ca518cd865 + +replace github.com/openstack-k8s-operators/keystone-operator/api => github.com/stuggi/keystone-operator/api v0.0.0-20230912050541-29ec0f576466 diff --git a/api/go.sum b/api/go.sum index b73f9120..9ac6a43b 100644 --- a/api/go.sum +++ b/api/go.sum @@ -222,8 +222,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230824094610-976b18ca2875 h1:pj22n6PQy/XAmV5m6XaarMY6X1lvxAh16oVT5ZSVoNI= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230824094610-976b18ca2875/go.mod h1:Mqg9hyHpWPda62750vqmk5TajxP3zbYPDP1rtSH7mg0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -278,6 +276,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stuggi/lib-common/modules/common v0.0.0-20230911170028-26ca518cd865 h1:6KbPTZCQRwZ/dUKLkyyIaKjT2HcMQDqjEcBQjEPXRv0= +github.com/stuggi/lib-common/modules/common v0.0.0-20230911170028-26ca518cd865/go.mod h1:Mqg9hyHpWPda62750vqmk5TajxP3zbYPDP1rtSH7mg0= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= diff --git a/api/v1beta1/horizon_types.go b/api/v1beta1/horizon_types.go index a87810ee..862cbbc6 100644 --- a/api/v1beta1/horizon_types.go +++ b/api/v1beta1/horizon_types.go @@ -20,6 +20,7 @@ import ( "fmt" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/util" corev1 "k8s.io/api/core/v1" @@ -80,16 +81,21 @@ type HorizonSpec struct { // https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ Resources corev1.ResourceRequirements `json:"resources,omitempty"` - // +kubebuilder:validation:Optional - // HorizonRoute holds all of the necessary options for configuring the Horizon Route object. - // This can be used to configure TLS - //TODO(bshephar) Implement everything about this. It's just a placeholder at the moment. - Route HorizonRoute `json:"route,omitempty"` - // +kubebuilder:validation:Required // +kubebuilder:default=memcached // Memcached instance name. MemcachedInstance string `json:"memcachedInstance"` + + // +kubebuilder:validation:Optional + // Override, provides the ability to override the generated manifest of several child resources. + Override HorizionOverrideSpec `json:"override,omitempty"` +} + +// HorizionOverrideSpec to override the generated manifest of several child resources. +type HorizionOverrideSpec struct { + // Override configuration for the Service created to serve traffic to the cluster. + // The key must be the endpoint type (public, internal) + Service map[string]service.RoutedOverrideSpec `json:"service,omitempty"` } // HorizonDebug can be used to enable debug in the Horizon service @@ -100,30 +106,6 @@ type HorizonDebug struct { Service bool `json:"service"` } -// HorizonRoute is used to define all of the information for the OpenShift route -// todo(bshephar) implement -type HorizonRoute struct { - // +kubebuilder:validation:Optional - // +kubebuilder:default=horizon - RouteName string `json:"routeName"` - - //TODO(bshephar) We need to implement TLS handling here to secure the route - // +kubebuilder:validation:Optional - RouteTLSEnabled string `json:"routeTLSEnabled,omitempty"` - - //TODO(bshephar) We need to implement TLS handling here to secure the route - // +kubebuilder:validation:Optional - RouteTLSCA string `json:"routeTLSCA,omitempty"` - - //TODO(bshephar) We need to implement TLS handling here to secure the route - // +kubebuilder:validation:Optional - RouteTLSKey string `json:"routeTLSKey,omitempty"` - - //TODO(bshephar) We need to implement TLS handling here to secure the route - // +kubebuilder:validation:Optional - RouteLocation string `json:"routeLocation,omitempty"` -} - // HorizonStatus defines the observed state of Horizon type HorizonStatus struct { // ReadyCount of Horizon instances diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index b9d9dd43..87378cf7 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -23,9 +23,32 @@ package v1beta1 import ( "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HorizionOverrideSpec) DeepCopyInto(out *HorizionOverrideSpec) { + *out = *in + if in.Service != nil { + in, out := &in.Service, &out.Service + *out = make(map[string]service.RoutedOverrideSpec, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizionOverrideSpec. +func (in *HorizionOverrideSpec) DeepCopy() *HorizionOverrideSpec { + if in == nil { + return nil + } + out := new(HorizionOverrideSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Horizon) DeepCopyInto(out *Horizon) { *out = *in @@ -115,21 +138,6 @@ func (in *HorizonList) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *HorizonRoute) DeepCopyInto(out *HorizonRoute) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizonRoute. -func (in *HorizonRoute) DeepCopy() *HorizonRoute { - if in == nil { - return nil - } - out := new(HorizonRoute) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HorizonSpec) DeepCopyInto(out *HorizonSpec) { *out = *in @@ -154,7 +162,7 @@ func (in *HorizonSpec) DeepCopyInto(out *HorizonSpec) { } } in.Resources.DeepCopyInto(&out.Resources) - out.Route = in.Route + in.Override.DeepCopyInto(&out.Override) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizonSpec. diff --git a/config/crd/bases/horizon.openstack.org_horizons.yaml b/config/crd/bases/horizon.openstack.org_horizons.yaml index d7bb6b54..afde2584 100644 --- a/config/crd/bases/horizon.openstack.org_horizons.yaml +++ b/config/crd/bases/horizon.openstack.org_horizons.yaml @@ -83,6 +83,180 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: RoutedOverrideSpec - a routed service override + configuration for the Service created to serve traffic to + the cluster. Allows for the manifest of the created Service + to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: EmbeddedLabelsAnnotations is an embedded subset + of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value + map stored with a resource that may be set by external + tools to store and retrieve arbitrary metadata. They + are not queryable and should be preserved when modifying + objects. More info: http://kubernetes.io/docs/user-guide/annotations' + type: object + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can + be used to organize and categorize (scope and select) + objects. May match selectors of replication controllers + and services. More info: http://kubernetes.io/docs/user-guide/labels' + type: object + type: object + spec: + description: OverrideServiceSpec is a subset of the fields + included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, + ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: externalName is the external reference + that discovery mechanisms will return as an alias + for this service (e.g. a DNS CNAME record). No proxying + will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires + `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: externalTrafficPolicy describes how nodes + distribute service traffic they receive on one of + the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", + the proxy will configure the service in a way that + assumes that external load balancers will take care + of balancing the service traffic between nodes, and + so each node will deliver traffic only to the node-local + endpoints of the service, without masquerading the + client source IP. (Traffic mistakenly sent to a node + with no endpoints will be dropped.) The default value, + "Cluster", uses the standard behavior of routing to + all endpoints evenly (possibly modified by topology + and other features). Note that traffic sent to an + External IP or LoadBalancer IP from within the cluster + will always get "Cluster" semantics, but clients sending + to a NodePort from within the cluster may need to + take traffic policy into account when picking a node. + type: string + internalTrafficPolicy: + description: InternalTrafficPolicy describes how nodes + distribute service traffic they receive on the ClusterIP. + If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the + same node as the pod, dropping the traffic if there + are no local endpoints. The default value, "Cluster", + uses the standard behavior of routing to all endpoints + evenly (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: IPFamilyPolicy represents the dual-stack-ness + requested or required by this Service. If there is + no value provided, then this field will be set to + SingleStack. Services can be "SingleStack" (a single + IP family), "PreferDualStack" (two IP families on + dual-stack configured clusters or a single IP family + on single-stack clusters), or "RequireDualStack" (two + IP families on dual-stack configured clusters, otherwise + fail). The ipFamilies and clusterIPs fields depend + on the value of this field. This field will be wiped + when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: loadBalancerClass is the class of the load + balancer implementation this Service belongs to. If + specified, the value of this field must be a label-style + identifier, with an optional prefix, e.g. "internal-vip" + or "example.com/internal-vip". Unprefixed names are + reserved for end-users. This field can only be set + when the Service type is 'LoadBalancer'. If not set, + the default load balancer implementation is used, + today this is typically done through the cloud provider + integration, but should apply for any default implementation. + If set, it is assumed that a load balancer implementation + is watching for Services with a matching class. Any + default load balancer implementation (e.g. cloud providers) + should ignore Services that set this field. This field + can only be set when creating or updating a Service + to type 'LoadBalancer'. Once set, it can not be changed. + This field will be wiped when a service is updated + to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified + client IPs. This field will be ignored if the cloud-provider + does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/' + items: + type: string + type: array + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to + maintain session affinity. Enable client IP based + session affinity. Must be ClientIP or None. Defaults + to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The + value must be >0 && <=86400(for 1 day) if + ServiceAffinity == "ClientIP". Default value + is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, + ClusterIP, NodePort, and LoadBalancer. "ClusterIP" + allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector + or if that is not specified, by manual construction + of an Endpoints object or EndpointSlice objects. If + clusterIP is "None", no virtual IP is allocated and + the endpoints are published as a set of endpoints + rather than a virtual IP. "NodePort" builds on ClusterIP + and allocates a port on every node which routes to + the same endpoints as the clusterIP. "LoadBalancer" + builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to + the same endpoints as the clusterIP. "ExternalName" + aliases this service to the specified externalName. + Several other fields do not apply to ExternalName + services. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + type: object + description: Override configuration for the Service created to + serve traffic to the cluster. The key must be the endpoint type + (public, internal) + type: object + type: object preserveJobs: default: false description: PreserveJobs - do not delete jobs after they finished @@ -143,32 +317,6 @@ spec: to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object - route: - description: HorizonRoute holds all of the necessary options for configuring - the Horizon Route object. This can be used to configure TLS TODO(bshephar) - Implement everything about this. It's just a placeholder at the - moment. - properties: - routeLocation: - description: TODO(bshephar) We need to implement TLS handling - here to secure the route - type: string - routeName: - default: horizon - type: string - routeTLSCA: - description: TODO(bshephar) We need to implement TLS handling - here to secure the route - type: string - routeTLSEnabled: - description: TODO(bshephar) We need to implement TLS handling - here to secure the route - type: string - routeTLSKey: - description: TODO(bshephar) We need to implement TLS handling - here to secure the route - type: string - type: object secret: description: Secret containing OpenStack password information for Horizon Secret Key diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index cbf28dc3..30bdcd96 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -145,18 +145,6 @@ rules: - list - update - watch -- apiGroups: - - route.openshift.io - resources: - - routes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - security.openshift.io resourceNames: diff --git a/controllers/horizon_controller.go b/controllers/horizon_controller.go index 8b6bed57..4f6ab29f 100644 --- a/controllers/horizon_controller.go +++ b/controllers/horizon_controller.go @@ -23,7 +23,6 @@ import ( "time" "github.com/go-logr/logr" - routev1 "github.com/openshift/api/route/v1" horizonv1beta1 "github.com/openstack-k8s-operators/horizon-operator/api/v1beta1" horizon "github.com/openstack-k8s-operators/horizon-operator/pkg/horizon" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" @@ -38,6 +37,7 @@ import ( labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" oko_secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" appsv1 "k8s.io/api/apps/v1" @@ -91,7 +91,6 @@ type HorizonReconciler struct { //+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete; //+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete; //+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete; -//+kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete; //+kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch; //+kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch; //+kubebuilder:rbac:groups=memcached.openstack.org,resources=memcacheds,verbs=get;list;watch; @@ -237,7 +236,6 @@ func (r *HorizonReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Secret{}). Owns(&corev1.ConfigMap{}). Owns(&appsv1.Deployment{}). - Owns(&routev1.Route{}). Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). @@ -267,43 +265,110 @@ func (r *HorizonReconciler) reconcileInit( l.Info("Reconciling Service init") // - // expose the service (create service, route and return the created endpoint URLs) + // expose the service (create service and return the created endpoint URLs) // - var horizonPorts = map[endpoint.Endpoint]endpoint.Data{ - endpoint.EndpointPublic: { + var horizonEndpoints = map[service.Endpoint]endpoint.Data{ + service.EndpointPublic: { Port: horizon.HorizonPublicPort, }, } - apiEndpoints, ctrlResult, err := endpoint.ExposeEndpoints( - ctx, - helper, - horizon.ServiceName, - serviceLabels, - horizonPorts, - time.Second * 5, - ) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.ExposeServiceReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.ExposeServiceReadyErrorMessage, - err.Error())) - return ctrlResult, err - } - if (ctrlResult != ctrl.Result{}) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.ExposeServiceReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.ExposeServiceReadyRunningMessage)) - return ctrlResult, nil + apiEndpoints := make(map[string]string) + + for endpointType, data := range horizonEndpoints { + endpointTypeStr := string(endpointType) + endpointName := horizon.ServiceName + "-" + endpointTypeStr + + svcOverride := instance.Spec.Override.Service[endpointTypeStr] + if svcOverride.EmbeddedLabelsAnnotations == nil { + svcOverride.EmbeddedLabelsAnnotations = &service.EmbeddedLabelsAnnotations{} + } + + exportLabels := util.MergeStringMaps( + serviceLabels, + map[string]string{ + service.AnnotationEndpointKey: endpointTypeStr, + }, + ) + + // Create the service + svc, err := service.NewService( + service.GenericService(&service.GenericServiceDetails{ + Name: endpointName, + Namespace: instance.Namespace, + Labels: exportLabels, + Selector: serviceLabels, + Port: service.GenericServicePort{ + Name: endpointName, + Port: data.Port, + Protocol: corev1.ProtocolTCP, + }, + }), + 5, + &svcOverride.OverrideSpec, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + + svc.AddAnnotation(map[string]string{ + service.AnnotationEndpointKey: endpointTypeStr, + }) + + // add Annotation to whether creating an ingress is required or not + if endpointType == service.EndpointPublic && svc.GetServiceType() == corev1.ServiceTypeClusterIP { + svc.AddAnnotation(map[string]string{ + service.AnnotationIngressCreateKey: "true", + }) + } else { + svc.AddAnnotation(map[string]string{ + service.AnnotationIngressCreateKey: "false", + }) + if svc.GetServiceType() == corev1.ServiceTypeLoadBalancer { + svc.AddAnnotation(map[string]string{ + service.AnnotationHostnameKey: svc.GetServiceHostname(), // add annotation to register service name in dnsmasq + }) + } + } + + ctrlResult, err := svc.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error())) + + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.ExposeServiceReadyRunningMessage)) + return ctrlResult, nil + } + // create service - end + + // TODO: TLS, pass in https as protocol, create TLS cert + apiEndpoints[string(endpointType)], err = svc.GetAPIEndpoint( + svcOverride.EndpointURL, data.Protocol, data.Path) + if err != nil { + return ctrl.Result{}, err + } } instance.Status.Conditions.MarkTrue(condition.ExposeServiceReadyCondition, condition.ExposeServiceReadyMessage) // - // Update instance status with service endpoint url from route host information + // Update instance status with service endpoint url information // instance.Status.Endpoint = apiEndpoints[string(endpoint.EndpointPublic)] @@ -489,7 +554,7 @@ func (r *HorizonReconciler) reconcileNormal(ctx context.Context, instance *horiz depl := deployment.NewDeployment( deplDef, - time.Second * 5, + time.Second*5, ) ctrlResult, err = depl.CreateOrPatch(ctx, helper) diff --git a/go.mod b/go.mod index 0a5dd039..be76ffeb 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/google/uuid v1.3.1 github.com/onsi/ginkgo/v2 v2.12.0 github.com/onsi/gomega v1.27.10 - github.com/openshift/api v3.9.0+incompatible github.com/openstack-k8s-operators/horizon-operator/api v0.1.1-0.20230828060631-f5678c16313e github.com/openstack-k8s-operators/infra-operator/apis v0.1.1-0.20230905074428-c6aefc16dd01 github.com/openstack-k8s-operators/keystone-operator/api v0.1.1-0.20230830083045-d73d07cca617 @@ -57,6 +56,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/openshift/api v3.9.0+incompatible // indirect github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20230824094610-976b18ca2875 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.14.0 // indirect @@ -97,3 +97,7 @@ replace github.com/openshift/api => github.com/openshift/api v0.0.0-202304141430 // needed to to cert-manager v1.11.4 see https://github.com/cert-manager/cert-manager/blob/v1.11.4/go.mod#L263C1-L264C104 // remove this once we bump to cert-manager v1.12.x replace github.com/Venafi/vcert/v4 => github.com/jetstack/vcert/v4 v4.9.6-0.20230519122548-219f317ae107 //allow-merging + +replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/stuggi/lib-common/modules/common v0.0.0-20230911170028-26ca518cd865 + +replace github.com/openstack-k8s-operators/keystone-operator/api => github.com/stuggi/keystone-operator/api v0.0.0-20230912050541-29ec0f576466 diff --git a/go.sum b/go.sum index 2774947b..dd7dc276 100644 --- a/go.sum +++ b/go.sum @@ -238,10 +238,6 @@ github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 h1:rncLxJBpFGqBztyxC github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7/go.mod h1:ctXNyWanKEjGj8sss1KjjHQ3ENKFm33FFnS5BKaIPh4= github.com/openstack-k8s-operators/infra-operator/apis v0.1.1-0.20230905074428-c6aefc16dd01 h1:Edoxiq+Tawt4P14oyuSGTCng23vcmKy4Hi68jMcpP14= github.com/openstack-k8s-operators/infra-operator/apis v0.1.1-0.20230905074428-c6aefc16dd01/go.mod h1:14D4OyzE7A5S3IkfXhU3B43mobM4QfdyQCCqLNLR4No= -github.com/openstack-k8s-operators/keystone-operator/api v0.1.1-0.20230830083045-d73d07cca617 h1:+Sz5kbF6QzXgKROXp3S+X22ZHdt68zvj1tBOu94OGKU= -github.com/openstack-k8s-operators/keystone-operator/api v0.1.1-0.20230830083045-d73d07cca617/go.mod h1:CmUe4tHh990eRUj6Ou8gD9JE0PQ38LGnUu3kaaP8K50= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230824094610-976b18ca2875 h1:pj22n6PQy/XAmV5m6XaarMY6X1lvxAh16oVT5ZSVoNI= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230824094610-976b18ca2875/go.mod h1:Mqg9hyHpWPda62750vqmk5TajxP3zbYPDP1rtSH7mg0= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20230824094610-976b18ca2875 h1:aUlwELsLYWQ3FL+/nRG/1uGVNW86c3MhtLrHNVDd57k= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20230824094610-976b18ca2875/go.mod h1:Vng+vqdTJUuZ+AEzSAaU0I7bn3qwYMMFEUHHhiH0440= github.com/openstack-k8s-operators/lib-common/modules/test v0.1.2-0.20230824094610-976b18ca2875 h1:Q23dbdnbcL5A54Ixb7oKVTRE24IcxiqizrZX7X43XOI= @@ -304,6 +300,10 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stuggi/keystone-operator/api v0.0.0-20230912050541-29ec0f576466 h1:vaDsV9XzjA8B24YHolxu2TRhjLfkR3t4t54uogmSksw= +github.com/stuggi/keystone-operator/api v0.0.0-20230912050541-29ec0f576466/go.mod h1:VJ9FF+3b7Cr2Fih5lqq8l8bKgDtWqnIO2g1C3vuqUug= +github.com/stuggi/lib-common/modules/common v0.0.0-20230911170028-26ca518cd865 h1:6KbPTZCQRwZ/dUKLkyyIaKjT2HcMQDqjEcBQjEPXRv0= +github.com/stuggi/lib-common/modules/common v0.0.0-20230911170028-26ca518cd865/go.mod h1:Mqg9hyHpWPda62750vqmk5TajxP3zbYPDP1rtSH7mg0= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= diff --git a/main.go b/main.go index 0f4585c2..4654c406 100644 --- a/main.go +++ b/main.go @@ -34,7 +34,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" - routev1 "github.com/openshift/api/route/v1" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" @@ -52,13 +51,9 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(horizonv1beta1.AddToScheme(scheme)) - //+kubebuilder:scaffold:scheme - - // As we are creating routes, it's necessary to register the routev1.Route - // type with the scheme - utilruntime.Must(routev1.AddToScheme(scheme)) utilruntime.Must(keystonev1.AddToScheme(scheme)) utilruntime.Must(memcachedv1.AddToScheme(scheme)) + //+kubebuilder:scaffold:scheme } func main() { diff --git a/tests/functional/horizon_controller_test.go b/tests/functional/horizon_controller_test.go index e86d72d6..418e3fe0 100644 --- a/tests/functional/horizon_controller_test.go +++ b/tests/functional/horizon_controller_test.go @@ -212,10 +212,6 @@ var _ = Describe("Horizon controller", func() { ContainSubstring("OPENSTACK_KEYSTONE_URL = \"http://keystone-public-openstack.testing/v3\"")) Expect(cm.Data["local_settings.py"]).Should( ContainSubstring("'LOCATION': [ 'memcached-0.memcached:11211', 'memcached-1.memcached:11211', 'memcached-2.memcached:11211' ]")) - th.AssertRouteExists(types.NamespacedName{ - Name: "horizon-public", - Namespace: horizonName.Namespace, - }) }) }) diff --git a/tests/functional/suite_test.go b/tests/functional/suite_test.go index b1e6fe4e..dcf53aaa 100644 --- a/tests/functional/suite_test.go +++ b/tests/functional/suite_test.go @@ -37,8 +37,6 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - routev1 "github.com/openshift/api/route/v1" - horizonv1 "github.com/openstack-k8s-operators/horizon-operator/api/v1beta1" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" @@ -88,8 +86,6 @@ var _ = BeforeSuite(func() { memcachedCRDs, err := test.GetCRDDirFromModule( "github.com/openstack-k8s-operators/infra-operator/apis", "../../go.mod", "bases") Expect(err).ShouldNot(HaveOccurred()) - routev1CRDs, err := test.GetOpenShiftCRDDir("route/v1", "../../go.mod") - Expect(err).ShouldNot(HaveOccurred()) By("bootstrapping test environment") testEnv = &envtest.Environment{ @@ -97,7 +93,6 @@ var _ = BeforeSuite(func() { filepath.Join("..", "..", "config", "crd", "bases"), keystoneCRDs, memcachedCRDs, - routev1CRDs, }, ErrorIfCRDPathMissing: true, WebhookInstallOptions: envtest.WebhookInstallOptions{ @@ -120,8 +115,6 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) err = memcachedv1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - err = routev1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) //+kubebuilder:scaffold:scheme