Skip to content

Commit

Permalink
feat: metrics aggregate (#269)
Browse files Browse the repository at this point in the history
* metrics aggregate

* add DebounceReconciler

* init global metrics

* add unit tests

* fix unit test

* fix after rebase
  • Loading branch information
zreigz authored Sep 26, 2024
1 parent de0d62f commit 22f31a5
Show file tree
Hide file tree
Showing 18 changed files with 1,453 additions and 12 deletions.
3 changes: 3 additions & 0 deletions .mockery.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,6 @@ packages:
github.com/pluralsh/deployment-operator/pkg/client:
interfaces:
Client:
k8s.io/client-go/discovery:
interfaces:
DiscoveryInterface:
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,10 @@ crd-ref-docs: --tool
controller-gen: TOOL = controller-gen
controller-gen: --tool

.PHONY: discovery
discovery: TOOL = discovery
discovery: --tool

# go-get-tool will 'go get' any package $2 and install it to $1.
PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
define go-get-tool
Expand Down
64 changes: 64 additions & 0 deletions api/v1alpha1/metricsaggregate_types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package v1alpha1

import (
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func init() {
SchemeBuilder.Register(&MetricsAggregate{}, &MetricsAggregateList{})
}

// MetricsAggregateList contains a list of [MetricsAggregate]
// +kubebuilder:object:root=true
type MetricsAggregateList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []MetricsAggregate `json:"items"`
}

// MetricsAggregate
// +kubebuilder:object:root=true
// +kubebuilder:resource:scope=Cluster
// +kubebuilder:subresource:status
type MetricsAggregate struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`

// Status of the MetricsAggregate
// +kubebuilder:validation:Optional
// +kubebuilder:printcolumn:name="Nodes",type=integer,JSONPath=".status.nodes",description="Number of Cluster Nodes"
// +kubebuilder:printcolumn:name="MemoryTotalBytes",type=integer,JSONPath=".status.memoryTotalBytes",description="Memory total bytes"
// +kubebuilder:printcolumn:name="MemoryAvailableBytes",type=integer,JSONPath=".status.memoryAvailableBytes",description="Memory available bytes"
// +kubebuilder:printcolumn:name="MemoryUsedPercentage",type=integer,JSONPath=".status.memoryUsedPercentage",description="Memory used percentage"
// +kubebuilder:printcolumn:name="CPUTotalMillicores",type=integer,JSONPath=".status.cpuTotalMillicores",description="CPU total millicores"
// +kubebuilder:printcolumn:name="CPUAvailableMillicores",type=integer,JSONPath=".status.cpuAvailableMillicores",description="CPU available millicores"
// +kubebuilder:printcolumn:name="CPUUsedPercentage",type=integer,JSONPath=".status.cpuUsedPercentage",description="CPU used percentage"
Status MetricsAggregateStatus `json:"status,omitempty"`
}

type MetricsAggregateStatus struct {
Nodes int `json:"nodes,omitempty"`
// MemoryTotalBytes current memory usage in bytes
MemoryTotalBytes int64 `json:"memoryTotalBytes,omitempty"`
// MemoryAvailableBytes available memory for node
MemoryAvailableBytes int64 `json:"memoryAvailableBytes,omitempty"`
// MemoryUsedPercentage in percentage
MemoryUsedPercentage int64 `json:"memoryUsedPercentage,omitempty"`
// CPUTotalMillicores in m cores
CPUTotalMillicores int64 `json:"cpuTotalMillicores,omitempty"`
// CPUAvailableMillicores in m cores
CPUAvailableMillicores int64 `json:"cpuAvailableMillicores,omitempty"`
// CPUUsedPercentage in percentage
CPUUsedPercentage int64 `json:"cpuUsedPercentage,omitempty"`

// +patchMergeKey=type
// +patchStrategy=merge
// +listType=map
// +listMapKey=type
Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
}

func (in *MetricsAggregate) SetCondition(condition metav1.Condition) {
meta.SetStatusCondition(&in.Status.Conditions, condition)
}
80 changes: 80 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.16.3
name: metricsaggregates.deployments.plural.sh
spec:
group: deployments.plural.sh
names:
kind: MetricsAggregate
listKind: MetricsAggregateList
plural: metricsaggregates
singular: metricsaggregate
scope: Cluster
versions:
- name: v1alpha1
schema:
openAPIV3Schema:
description: MetricsAggregate
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
status:
description: Status of the MetricsAggregate
properties:
conditions:
items:
description: Condition contains details for one aspect of the current
state of this API Resource.
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase.
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
cpuAvailableMillicores:
description: CPUAvailableMillicores in m cores
format: int64
type: integer
cpuTotalMillicores:
description: CPUTotalMillicores in m cores
format: int64
type: integer
cpuUsedPercentage:
description: CPUUsedPercentage in percentage
format: int64
type: integer
memoryAvailableBytes:
description: MemoryAvailableBytes available memory for node
format: int64
type: integer
memoryTotalBytes:
description: MemoryTotalBytes current memory usage in bytes
format: int64
type: integer
memoryUsedPercentage:
description: MemoryUsedPercentage in percentage
format: int64
type: integer
nodes:
type: integer
type: object
type: object
served: true
storage: true
subresources:
status: {}
12 changes: 12 additions & 0 deletions cmd/agent/kubernetes.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package main

import (
"context"
"net/http"
"os"
"strings"
Expand All @@ -11,6 +12,7 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/discovery"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
Expand Down Expand Up @@ -85,10 +87,12 @@ func initKubeClientsOrDie(config *rest.Config) (rolloutsClient *roclientset.Clie
}

func registerKubeReconcilersOrDie(
ctx context.Context,
manager ctrl.Manager,
consoleManager *consolectrl.Manager,
config *rest.Config,
extConsoleClient consoleclient.Client,
discoveryClient discovery.DiscoveryInterface,
) {
rolloutsClient, dynamicClient, kubeClient := initKubeClientsOrDie(config)

Expand Down Expand Up @@ -208,4 +212,12 @@ func registerKubeReconcilersOrDie(
setupLog.Error(err, "unable to create controller", "controller", "Group")
os.Exit(1)
}

if err := (&controller.MetricsAggregateReconciler{
Client: manager.GetClient(),
Scheme: manager.GetScheme(),
DiscoveryClient: discoveryClient,
}).SetupWithManager(ctx, manager); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "MetricsAggregate")
}
}
15 changes: 8 additions & 7 deletions cmd/agent/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,22 +8,22 @@ import (
rolloutv1alpha1 "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
templatesv1 "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1"
constraintstatusv1beta1 "github.com/open-policy-agent/gatekeeper/v3/apis/status/v1beta1"
deploymentsv1alpha1 "github.com/pluralsh/deployment-operator/api/v1alpha1"
"github.com/pluralsh/deployment-operator/cmd/agent/args"
"github.com/pluralsh/deployment-operator/pkg/cache"
"github.com/pluralsh/deployment-operator/pkg/client"
consolectrl "github.com/pluralsh/deployment-operator/pkg/controller"
"k8s.io/client-go/discovery"
"k8s.io/client-go/rest"
"k8s.io/klog/v2"
metricsv1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1"

velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"

deploymentsv1alpha1 "github.com/pluralsh/deployment-operator/api/v1alpha1"
"github.com/pluralsh/deployment-operator/cmd/agent/args"
"github.com/pluralsh/deployment-operator/pkg/cache"
"github.com/pluralsh/deployment-operator/pkg/client"
consolectrl "github.com/pluralsh/deployment-operator/pkg/controller"
)

var (
Expand All @@ -39,6 +39,7 @@ func init() {
utilruntime.Must(constraintstatusv1beta1.AddToScheme(scheme))
utilruntime.Must(templatesv1.AddToScheme(scheme))
utilruntime.Must(rolloutv1alpha1.AddToScheme(scheme))
utilruntime.Must(metricsv1beta1.AddToScheme(scheme))
//+kubebuilder:scaffold:scheme
}

Expand All @@ -60,7 +61,7 @@ func main() {
cache.InitGateCache(args.ControllerCacheTTL(), extConsoleClient)

registerConsoleReconcilersOrDie(consoleManager, config, kubeManager.GetClient(), extConsoleClient)
registerKubeReconcilersOrDie(kubeManager, consoleManager, config, extConsoleClient)
registerKubeReconcilersOrDie(ctx, kubeManager, consoleManager, config, extConsoleClient, discoveryClient)

//+kubebuilder:scaffold:builder

Expand Down
Loading

0 comments on commit 22f31a5

Please sign in to comment.