diff --git a/api/v1alpha1/metricsaggregate_types.go b/api/v1alpha1/metricsaggregate_types.go new file mode 100644 index 00000000..113daf51 --- /dev/null +++ b/api/v1alpha1/metricsaggregate_types.go @@ -0,0 +1,55 @@ +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func init() { + SchemeBuilder.Register(&MetricsAggregate{}, &MetricsAggregateList{}) +} + +// MetricsAggregateList contains a list of [MetricsAggregate] +// +kubebuilder:object:root=true +type MetricsAggregateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []MetricsAggregate `json:"items"` +} + +// MetricsAggregate +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:subresource:status +type MetricsAggregate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Spec of the MetricsAggregate + // +kubebuilder:validation:Required + Spec MetricsAggregateSpec `json:"spec"` + + // Status of the IngressReplica + // +kubebuilder:validation:Optional + Status Status `json:"status,omitempty"` +} + +type MetricsAggregateSpec struct { + Nodes int `json:"nodes"` + // MemoryTotalBytes current memory usage in bytes + MemoryTotalBytes int64 `json:"memoryTotalBytes,omitempty"` + // MemoryAvailableBytes available memory for node + MemoryAvailableBytes int64 `json:"memoryAvailableBytes,omitempty"` + // MemoryUsedPercentage in percentage + MemoryUsedPercentage int64 `json:"memoryUsedPercentage,omitempty"` + // CPUTotalMillicores in m cores + CPUTotalMillicores int64 `json:"cpuTotalMillicores,omitempty"` + // CPUAvailableMillicores in m cores + CPUAvailableMillicores int64 `json:"cpuAvailableMillicores,omitempty"` + // CPUUsedPercentage in percentage + CPUUsedPercentage int64 `json:"cpuUsedPercentage,omitempty"` +} + +func (in *MetricsAggregate) SetCondition(condition metav1.Condition) { + meta.SetStatusCondition(&in.Status.Conditions, condition) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6cc6cc1d..b903b9d0 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -347,6 +347,80 @@ func (in *HelmSpec) DeepCopy() *HelmSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricsAggregate) DeepCopyInto(out *MetricsAggregate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsAggregate. +func (in *MetricsAggregate) DeepCopy() *MetricsAggregate { + if in == nil { + return nil + } + out := new(MetricsAggregate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MetricsAggregate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricsAggregateList) DeepCopyInto(out *MetricsAggregateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]MetricsAggregate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsAggregateList. +func (in *MetricsAggregateList) DeepCopy() *MetricsAggregateList { + if in == nil { + return nil + } + out := new(MetricsAggregateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MetricsAggregateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricsAggregateSpec) DeepCopyInto(out *MetricsAggregateSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsAggregateSpec. +func (in *MetricsAggregateSpec) DeepCopy() *MetricsAggregateSpec { + if in == nil { + return nil + } + out := new(MetricsAggregateSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PipelineGate) DeepCopyInto(out *PipelineGate) { *out = *in diff --git a/charts/deployment-operator/crds/deployments.plural.sh_metricsaggregates.yaml b/charts/deployment-operator/crds/deployments.plural.sh_metricsaggregates.yaml new file mode 100644 index 00000000..6766888f --- /dev/null +++ b/charts/deployment-operator/crds/deployments.plural.sh_metricsaggregates.yaml @@ -0,0 +1,148 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: metricsaggregates.deployments.plural.sh +spec: + group: deployments.plural.sh + names: + kind: MetricsAggregate + listKind: MetricsAggregateList + plural: metricsaggregates + singular: metricsaggregate + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: MetricsAggregate + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec of the MetricsAggregate + properties: + cpuAvailableMillicores: + description: CPUAvailableMillicores in m cores + format: int64 + type: integer + cpuTotalMillicores: + description: CPUTotalMillicores in m cores + format: int64 + type: integer + cpuUsedPercentage: + description: CPUUsedPercentage in percentage + format: int64 + type: integer + memoryAvailableBytes: + description: MemoryAvailableBytes available memory for node + format: int64 + type: integer + memoryTotalBytes: + description: MemoryTotalBytes current memory usage in bytes + format: int64 + type: integer + memoryUsedPercentage: + description: MemoryUsedPercentage in percentage + format: int64 + type: integer + nodes: + type: integer + required: + - nodes + type: object + status: + description: Status of the IngressReplica + properties: + conditions: + description: Represents the observations of a PrAutomation's current + state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + id: + description: ID of the resource in the Console API. + type: string + sha: + description: SHA of last applied configuration. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/deployments.plural.sh_metricsaggregates.yaml b/config/crd/bases/deployments.plural.sh_metricsaggregates.yaml new file mode 100644 index 00000000..6766888f --- /dev/null +++ b/config/crd/bases/deployments.plural.sh_metricsaggregates.yaml @@ -0,0 +1,148 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: metricsaggregates.deployments.plural.sh +spec: + group: deployments.plural.sh + names: + kind: MetricsAggregate + listKind: MetricsAggregateList + plural: metricsaggregates + singular: metricsaggregate + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: MetricsAggregate + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec of the MetricsAggregate + properties: + cpuAvailableMillicores: + description: CPUAvailableMillicores in m cores + format: int64 + type: integer + cpuTotalMillicores: + description: CPUTotalMillicores in m cores + format: int64 + type: integer + cpuUsedPercentage: + description: CPUUsedPercentage in percentage + format: int64 + type: integer + memoryAvailableBytes: + description: MemoryAvailableBytes available memory for node + format: int64 + type: integer + memoryTotalBytes: + description: MemoryTotalBytes current memory usage in bytes + format: int64 + type: integer + memoryUsedPercentage: + description: MemoryUsedPercentage in percentage + format: int64 + type: integer + nodes: + type: integer + required: + - nodes + type: object + status: + description: Status of the IngressReplica + properties: + conditions: + description: Represents the observations of a PrAutomation's current + state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + id: + description: ID of the resource in the Console API. + type: string + sha: + description: SHA of last applied configuration. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/samples/metricsAggregate.yaml b/config/samples/metricsAggregate.yaml new file mode 100644 index 00000000..8455b56a --- /dev/null +++ b/config/samples/metricsAggregate.yaml @@ -0,0 +1,11 @@ +apiVersion: pipelines.plural.sh/v1alpha1 +kind: MetricsAggregate +metadata: + labels: + app.kubernetes.io/name: metrics-aggregate + app.kubernetes.io/instance: metricsaggregate-sample + app.kubernetes.io/part-of: deployment-operator + app.kubernetes.io/created-by: deployment-operator + name: metrics-aggregate-sample +spec: + nodes: 0 \ No newline at end of file diff --git a/go.mod b/go.mod index a9867de3..3d2dbe4d 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,6 @@ require ( github.com/yuin/gopher-lua v1.1.1 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 - golang.org/x/net v0.29.0 gopkg.in/yaml.v3 v3.0.1 helm.sh/helm/v3 v3.16.1 k8s.io/api v0.31.1 @@ -52,6 +51,7 @@ require ( k8s.io/client-go v0.31.1 k8s.io/klog/v2 v2.130.1 k8s.io/kubectl v0.31.1 + k8s.io/metrics v0.31.1 k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 layeh.com/gopher-luar v1.0.11 sigs.k8s.io/cli-utils v0.37.2 @@ -356,6 +356,7 @@ require ( golang.org/x/crypto v0.27.0 // indirect golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect golang.org/x/mod v0.21.0 // indirect + golang.org/x/net v0.29.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.25.0 // indirect diff --git a/go.sum b/go.sum index e493da48..5ddf5363 100644 --- a/go.sum +++ b/go.sum @@ -1149,6 +1149,8 @@ k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f h1:0LQagt0gDpKqvIkAMPaRGc k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f/go.mod h1:S9tOR0FxgyusSNR+MboCuiDpVWkAifZvaYI1Q2ubgro= k8s.io/kubectl v0.31.1 h1:ih4JQJHxsEggFqDJEHSOdJ69ZxZftgeZvYo7M/cpp24= k8s.io/kubectl v0.31.1/go.mod h1:aNuQoR43W6MLAtXQ/Bu4GDmoHlbhHKuyD49lmTC8eJM= +k8s.io/metrics v0.31.1 h1:h4I4dakgh/zKflWYAOQhwf0EXaqy8LxAIyE/GBvxqRc= +k8s.io/metrics v0.31.1/go.mod h1:JuH1S9tJiH9q1VCY0yzSCawi7kzNLsDzlWDJN4xR+iA= k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= layeh.com/gopher-luar v1.0.11 h1:8zJudpKI6HWkoh9eyyNFaTM79PY6CAPcIr6X/KTiliw= diff --git a/internal/controller/metricsaggregate.go b/internal/controller/metricsaggregate.go new file mode 100644 index 00000000..76c6de6a --- /dev/null +++ b/internal/controller/metricsaggregate.go @@ -0,0 +1,143 @@ +package controller + +import ( + "context" + "fmt" + + "github.com/pluralsh/deployment-operator/internal/utils" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/pluralsh/deployment-operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/metrics/pkg/apis/metrics/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" + k8sClient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// MetricsAggregateReconciler reconciles a MetricsAggregate resource. +type MetricsAggregateReconciler struct { + k8sClient.Client + Scheme *runtime.Scheme +} + +// Reconcile IngressReplica ensure that stays in sync with Kubernetes cluster. +func (r *MetricsAggregateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ reconcile.Result, reterr error) { + logger := log.FromContext(ctx) + + // Read resource from Kubernetes cluster. + metrics := &v1alpha1.MetricsAggregate{} + if err := r.Get(ctx, req.NamespacedName, metrics); err != nil { + logger.Error(err, "unable to fetch MetricsAggregate") + return ctrl.Result{}, k8sClient.IgnoreNotFound(err) + } + + logger.Info("reconciling MetricsAggregate", "namespace", metrics.Namespace, "name", metrics.Name) + utils.MarkCondition(metrics.SetCondition, v1alpha1.ReadyConditionType, metav1.ConditionFalse, v1alpha1.ReadyConditionReason, "") + + scope, err := NewDefaultScope(ctx, r.Client, metrics) + if err != nil { + logger.Error(err, "failed to create scope") + utils.MarkCondition(metrics.SetCondition, v1alpha1.ReadyConditionType, metav1.ConditionFalse, v1alpha1.ReadyConditionReason, err.Error()) + return ctrl.Result{}, err + } + + // Always patch object when exiting this function, so we can persist any object changes. + defer func() { + if err := scope.PatchObject(); err != nil && reterr == nil { + reterr = err + } + }() + + if !metrics.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + + nodeList := &corev1.NodeList{} + availableResources := make(map[string]corev1.ResourceList) + + for _, n := range nodeList.Items { + availableResources[n.Name] = n.Status.Allocatable + } + + nodeDeploymentNodesMetrics := make([]v1beta1.NodeMetrics, 0) + allNodeMetricsList := &v1beta1.NodeMetricsList{} + if err := r.List(ctx, allNodeMetricsList); err != nil { + return reconcile.Result{}, err + } + + for _, m := range allNodeMetricsList.Items { + if _, ok := availableResources[m.Name]; ok { + nodeDeploymentNodesMetrics = append(nodeDeploymentNodesMetrics, m) + } + } + + nodeMetrics, err := ConvertNodeMetrics(nodeDeploymentNodesMetrics, availableResources) + if err != nil { + return reconcile.Result{}, err + } + metrics.Spec.Nodes = len(nodeList.Items) + for _, nm := range nodeMetrics { + metrics.Spec.CPUAvailableMillicores += nm.CPUAvailableMillicores + metrics.Spec.CPUTotalMillicores += nm.CPUTotalMillicores + metrics.Spec.MemoryAvailableBytes += nm.MemoryAvailableBytes + metrics.Spec.MemoryTotalBytes += nm.MemoryTotalBytes + } + + fraction := float64(metrics.Spec.CPUTotalMillicores) / float64(metrics.Spec.CPUAvailableMillicores) * 100 + metrics.Spec.CPUUsedPercentage = int64(fraction) + fraction = float64(metrics.Spec.MemoryTotalBytes) / float64(metrics.Spec.MemoryAvailableBytes) * 100 + metrics.Spec.MemoryUsedPercentage = int64(fraction) + + return requeue(requeueAfter, jitter), reterr +} + +// SetupWithManager sets up the controller with the Manager. +func (r *MetricsAggregateReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.MetricsAggregate{}). + Complete(r) +} + +type ResourceMetricsInfo struct { + Name string + Metrics corev1.ResourceList + Available corev1.ResourceList +} + +func ConvertNodeMetrics(metrics []v1beta1.NodeMetrics, availableResources map[string]corev1.ResourceList) ([]v1alpha1.MetricsAggregateSpec, error) { + nodeMetrics := make([]v1alpha1.MetricsAggregateSpec, 0) + + if metrics == nil { + return nil, fmt.Errorf("metric list can not be nil") + } + + for _, m := range metrics { + nodeMetric := v1alpha1.MetricsAggregateSpec{} + + resourceMetricsInfo := ResourceMetricsInfo{ + Name: m.Name, + Metrics: m.Usage.DeepCopy(), + Available: availableResources[m.Name], + } + + if available, found := resourceMetricsInfo.Available[corev1.ResourceCPU]; found { + quantityCPU := resourceMetricsInfo.Metrics[corev1.ResourceCPU] + // cpu in mili cores + nodeMetric.CPUTotalMillicores = quantityCPU.MilliValue() + nodeMetric.CPUAvailableMillicores = available.MilliValue() + } + + if available, found := resourceMetricsInfo.Available[corev1.ResourceMemory]; found { + quantityM := resourceMetricsInfo.Metrics[corev1.ResourceMemory] + // memory in bytes + nodeMetric.MemoryTotalBytes = quantityM.Value() / (1024 * 1024) + nodeMetric.MemoryAvailableBytes = available.Value() / (1024 * 1024) + } + nodeMetrics = append(nodeMetrics, nodeMetric) + } + + return nodeMetrics, nil +}