From 65e7e685058737882acf7ce02eadbd4cc89c7195 Mon Sep 17 00:00:00 2001 From: Timo Riski Date: Mon, 28 Oct 2024 17:03:39 +0200 Subject: [PATCH 1/5] feat: generate userconfigs for autoscaler integration and integration endpoint --- .../integration/autoscaler/autoscaler.go | 7 +++ .../autoscaler/zz_generated.deepcopy.go | 22 +++++++++ .../autoscaler/autoscaler.go | 21 ++++++++ .../autoscaler/zz_generated.deepcopy.go | 48 +++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- main.go | 4 +- 7 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 api/v1alpha1/userconfig/integration/autoscaler/autoscaler.go create mode 100644 api/v1alpha1/userconfig/integration/autoscaler/zz_generated.deepcopy.go create mode 100644 api/v1alpha1/userconfig/integrationendpoints/autoscaler/autoscaler.go create mode 100644 api/v1alpha1/userconfig/integrationendpoints/autoscaler/zz_generated.deepcopy.go diff --git a/api/v1alpha1/userconfig/integration/autoscaler/autoscaler.go b/api/v1alpha1/userconfig/integration/autoscaler/autoscaler.go new file mode 100644 index 00000000..cb1fec22 --- /dev/null +++ b/api/v1alpha1/userconfig/integration/autoscaler/autoscaler.go @@ -0,0 +1,7 @@ +// Code generated by user config generator. DO NOT EDIT. +// +kubebuilder:object:generate=true + +package autoscaleruserconfig + +// Integration user config +type AutoscalerUserConfig struct{} diff --git a/api/v1alpha1/userconfig/integration/autoscaler/zz_generated.deepcopy.go b/api/v1alpha1/userconfig/integration/autoscaler/zz_generated.deepcopy.go new file mode 100644 index 00000000..671defb8 --- /dev/null +++ b/api/v1alpha1/userconfig/integration/autoscaler/zz_generated.deepcopy.go @@ -0,0 +1,22 @@ +//go:build !ignore_autogenerated + +// Copyright (c) 2024 Aiven, Helsinki, Finland. https://aiven.io/ + +// Code generated by controller-gen. DO NOT EDIT. + +package autoscaleruserconfig + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AutoscalerUserConfig) DeepCopyInto(out *AutoscalerUserConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalerUserConfig. +func (in *AutoscalerUserConfig) DeepCopy() *AutoscalerUserConfig { + if in == nil { + return nil + } + out := new(AutoscalerUserConfig) + in.DeepCopyInto(out) + return out +} diff --git a/api/v1alpha1/userconfig/integrationendpoints/autoscaler/autoscaler.go b/api/v1alpha1/userconfig/integrationendpoints/autoscaler/autoscaler.go new file mode 100644 index 00000000..46507204 --- /dev/null +++ b/api/v1alpha1/userconfig/integrationendpoints/autoscaler/autoscaler.go @@ -0,0 +1,21 @@ +// Code generated by user config generator. DO NOT EDIT. +// +kubebuilder:object:generate=true + +package autoscaleruserconfig + +// AutoscalingProperties +type Autoscaling struct { + // +kubebuilder:validation:Minimum=50 + // +kubebuilder:validation:Maximum=10000 + // The maximum total disk size (in gb) to allow autoscaler to scale up to + CapGb int `groups:"create,update" json:"cap_gb"` + + // +kubebuilder:validation:Enum="autoscale_disk" + // Type of autoscale event + Type string `groups:"create,update" json:"type"` +} +type AutoscalerUserConfig struct { + // +kubebuilder:validation:MaxItems=64 + // Configure autoscaling thresholds for a service + Autoscaling []*Autoscaling `groups:"create,update" json:"autoscaling"` +} diff --git a/api/v1alpha1/userconfig/integrationendpoints/autoscaler/zz_generated.deepcopy.go b/api/v1alpha1/userconfig/integrationendpoints/autoscaler/zz_generated.deepcopy.go new file mode 100644 index 00000000..51460669 --- /dev/null +++ b/api/v1alpha1/userconfig/integrationendpoints/autoscaler/zz_generated.deepcopy.go @@ -0,0 +1,48 @@ +//go:build !ignore_autogenerated + +// Copyright (c) 2024 Aiven, Helsinki, Finland. https://aiven.io/ + +// Code generated by controller-gen. DO NOT EDIT. + +package autoscaleruserconfig + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AutoscalerUserConfig) DeepCopyInto(out *AutoscalerUserConfig) { + *out = *in + if in.Autoscaling != nil { + in, out := &in.Autoscaling, &out.Autoscaling + *out = make([]*Autoscaling, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Autoscaling) + **out = **in + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalerUserConfig. +func (in *AutoscalerUserConfig) DeepCopy() *AutoscalerUserConfig { + if in == nil { + return nil + } + out := new(AutoscalerUserConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Autoscaling) DeepCopyInto(out *Autoscaling) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Autoscaling. +func (in *Autoscaling) DeepCopy() *Autoscaling { + if in == nil { + return nil + } + out := new(Autoscaling) + in.DeepCopyInto(out) + return out +} diff --git a/go.mod b/go.mod index 27b3bad4..4b108bc6 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/stoewer/go-strcase v1.3.0 github.com/stretchr/testify v1.9.0 github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 + golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c golang.org/x/sync v0.8.0 golang.org/x/tools v0.26.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 28b06166..b5acd71e 100644 --- a/go.sum +++ b/go.sum @@ -409,8 +409,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM= -golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/main.go b/main.go index f673b6c0..27252e06 100644 --- a/main.go +++ b/main.go @@ -25,8 +25,8 @@ import ( ) //go:generate go run ./generators/userconfigs/... --services mysql,cassandra,grafana,pg,kafka,redis,clickhouse,opensearch,kafka_connect -//go:generate go run ./generators/userconfigs/... --integrations clickhouse_kafka,clickhouse_postgresql,datadog,kafka_connect,kafka_logs,kafka_mirrormaker,logs,metrics,external_aws_cloudwatch_metrics -//go:generate go run ./generators/userconfigs/... --integration-endpoints datadog,external_aws_cloudwatch_logs,external_aws_cloudwatch_metrics,external_elasticsearch_logs,external_google_cloud_bigquery,external_google_cloud_logging,external_kafka,external_opensearch_logs,external_postgresql,external_schema_registry,jolokia,prometheus,rsyslog +//go:generate go run ./generators/userconfigs/... --integrations autoscaler,clickhouse_kafka,clickhouse_postgresql,datadog,kafka_connect,kafka_logs,kafka_mirrormaker,logs,metrics,external_aws_cloudwatch_metrics +//go:generate go run ./generators/userconfigs/... --integration-endpoints autoscaler,datadog,external_aws_cloudwatch_logs,external_aws_cloudwatch_metrics,external_elasticsearch_logs,external_google_cloud_bigquery,external_google_cloud_logging,external_kafka,external_opensearch_logs,external_postgresql,external_schema_registry,jolokia,prometheus,rsyslog var ( scheme = runtime.NewScheme() From 7ab38f8bd61c9a8409f8c251dc82962f93835e32 Mon Sep 17 00:00:00 2001 From: Timo Riski Date: Mon, 28 Oct 2024 17:13:26 +0200 Subject: [PATCH 2/5] feat: add `autoscaler` to api types --- api/v1alpha1/serviceintegration_types.go | 5 +++++ api/v1alpha1/serviceintegrationendpoint_types.go | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/api/v1alpha1/serviceintegration_types.go b/api/v1alpha1/serviceintegration_types.go index 2519496a..a3c8e20d 100644 --- a/api/v1alpha1/serviceintegration_types.go +++ b/api/v1alpha1/serviceintegration_types.go @@ -9,6 +9,7 @@ import ( "github.com/aiven/go-client-codegen/handler/service" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + autoscalerintegration "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integration/autoscaler" clickhousekafkauserconfig "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integration/clickhouse_kafka" clickhousepostgresqluserconfig "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integration/clickhouse_postgresql" datadogintegration "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integration/datadog" @@ -59,6 +60,9 @@ type ServiceIntegrationSpec struct { // Destination project for the integration (if any) DestinationProjectName string `json:"destinationProjectName,omitempty"` + // Autoscaler specific user configuration options + AutoscalerUserConfig *autoscalerintegration.AutoscalerUserConfig `json:"autoscaler,omitempty"` + // Datadog specific user configuration options DatadogUserConfig *datadogintegration.DatadogUserConfig `json:"datadog,omitempty"` @@ -130,6 +134,7 @@ func (in *ServiceIntegration) Conditions() *[]metav1.Condition { func (in *ServiceIntegration) getUserConfigFields() map[service.IntegrationType]any { return map[service.IntegrationType]any{ + service.IntegrationTypeAutoscaler: in.Spec.AutoscalerUserConfig, service.IntegrationTypeClickhouseKafka: in.Spec.ClickhouseKafkaUserConfig, service.IntegrationTypeClickhousePostgresql: in.Spec.ClickhousePostgreSQLUserConfig, service.IntegrationTypeDatadog: in.Spec.DatadogUserConfig, diff --git a/api/v1alpha1/serviceintegrationendpoint_types.go b/api/v1alpha1/serviceintegrationendpoint_types.go index c24e3627..de3349b2 100644 --- a/api/v1alpha1/serviceintegrationendpoint_types.go +++ b/api/v1alpha1/serviceintegrationendpoint_types.go @@ -8,6 +8,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + autoscaleruserconfig "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integrationendpoints/autoscaler" datadoguserconfig "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integrationendpoints/datadog" externalawscloudwatchlogsuserconfig "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integrationendpoints/external_aws_cloudwatch_logs" externalawscloudwatchmetricsuserconfig "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integrationendpoints/external_aws_cloudwatch_metrics" @@ -37,6 +38,9 @@ type ServiceIntegrationEndpointSpec struct { // Source endpoint for the integration (if any) EndpointName string `json:"endpointName,omitempty"` + // Autoscaler configuration values + Autoscaler *autoscaleruserconfig.AutoscalerUserConfig `json:"autoscaler,omitempty"` + // Datadog configuration values Datadog *datadoguserconfig.DatadogUserConfig `json:"datadog,omitempty"` @@ -118,6 +122,7 @@ func (in *ServiceIntegrationEndpoint) Conditions() *[]metav1.Condition { func (in *ServiceIntegrationEndpoint) getUserConfigFields() map[string]any { return map[string]any{ + "autoscaler": in.Spec.Autoscaler, "datadog": in.Spec.Datadog, "external_aws_cloudwatch_logs": in.Spec.ExternalAwsCloudwatchLogs, "external_aws_cloudwatch_metrics": in.Spec.ExternalAwsCloudwatchMetrics, From 291b2f89bbaf9e586dfd05369efb75d0992a41d5 Mon Sep 17 00:00:00 2001 From: Timo Riski Date: Mon, 28 Oct 2024 17:13:50 +0200 Subject: [PATCH 3/5] chore: generated code for `autoscaler` types Generated with `make generate` --- CHANGELOG.md | 2 ++ api/v1alpha1/zz_generated.deepcopy.go | 12 ++++++++ .../aiven.io_serviceintegrationendpoints.yaml | 29 +++++++++++++++++++ .../aiven.io_serviceintegrations.yaml | 3 ++ .../aiven.io_serviceintegrationendpoints.yaml | 29 +++++++++++++++++++ .../bases/aiven.io_serviceintegrations.yaml | 3 ++ docs/docs/api-reference/serviceintegration.md | 1 + .../serviceintegrationendpoint.md | 22 ++++++++++++++ 8 files changed, 101 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index de774d33..7c110539 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,8 @@ aliases alongside their associated indexes. Default is true - Add `OpenSearch` field `userConfig.s3_migration.include_aliases`, type `boolean`: Whether to restore aliases alongside their associated indexes. Default is true +- Add `ServiceIntegration` field `autoscaler`, type `object`: Autoscaler specific user configuration options +- Add `ServiceIntegrationEndpoint` field `autoscaler`, type `object`: Autoscaler configuration values ## v0.25.0 - 2024-09-19 diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 4f9480b4..bea6ad57 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -10,6 +10,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + autoscaler "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integration/autoscaler" clickhouse_kafka "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integration/clickhouse_kafka" clickhouse_postgresql "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integration/clickhouse_postgresql" datadog "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integration/datadog" @@ -19,6 +20,7 @@ import ( kafka_mirrormaker "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integration/kafka_mirrormaker" logs "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integration/logs" metrics "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integration/metrics" + integrationendpointsautoscaler "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integrationendpoints/autoscaler" integrationendpointsdatadog "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integrationendpoints/datadog" external_aws_cloudwatch_logs "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integrationendpoints/external_aws_cloudwatch_logs" integrationendpointsexternal_aws_cloudwatch_metrics "github.com/aiven/aiven-operator/api/v1alpha1/userconfig/integrationendpoints/external_aws_cloudwatch_metrics" @@ -2686,6 +2688,11 @@ func (in *ServiceIntegrationEndpointList) DeepCopyObject() runtime.Object { func (in *ServiceIntegrationEndpointSpec) DeepCopyInto(out *ServiceIntegrationEndpointSpec) { *out = *in in.ProjectDependant.DeepCopyInto(&out.ProjectDependant) + if in.Autoscaler != nil { + in, out := &in.Autoscaler, &out.Autoscaler + *out = new(integrationendpointsautoscaler.AutoscalerUserConfig) + (*in).DeepCopyInto(*out) + } if in.Datadog != nil { in, out := &in.Datadog, &out.Datadog *out = new(integrationendpointsdatadog.DatadogUserConfig) @@ -2836,6 +2843,11 @@ func (in *ServiceIntegrationList) DeepCopyObject() runtime.Object { func (in *ServiceIntegrationSpec) DeepCopyInto(out *ServiceIntegrationSpec) { *out = *in in.ProjectDependant.DeepCopyInto(&out.ProjectDependant) + if in.AutoscalerUserConfig != nil { + in, out := &in.AutoscalerUserConfig, &out.AutoscalerUserConfig + *out = new(autoscaler.AutoscalerUserConfig) + **out = **in + } if in.DatadogUserConfig != nil { in, out := &in.DatadogUserConfig, &out.DatadogUserConfig *out = new(datadog.DatadogUserConfig) diff --git a/charts/aiven-operator-crds/templates/aiven.io_serviceintegrationendpoints.yaml b/charts/aiven-operator-crds/templates/aiven.io_serviceintegrationendpoints.yaml index 2893db2a..679f0033 100644 --- a/charts/aiven-operator-crds/templates/aiven.io_serviceintegrationendpoints.yaml +++ b/charts/aiven-operator-crds/templates/aiven.io_serviceintegrationendpoints.yaml @@ -69,6 +69,35 @@ spec: - key - name type: object + autoscaler: + description: Autoscaler configuration values + properties: + autoscaling: + description: Configure autoscaling thresholds for a service + items: + description: AutoscalingProperties + properties: + cap_gb: + description: + The maximum total disk size (in gb) to allow + autoscaler to scale up to + maximum: 10000 + minimum: 50 + type: integer + type: + description: Type of autoscale event + enum: + - autoscale_disk + type: string + required: + - cap_gb + - type + type: object + maxItems: 64 + type: array + required: + - autoscaling + type: object datadog: description: Datadog configuration values properties: diff --git a/charts/aiven-operator-crds/templates/aiven.io_serviceintegrations.yaml b/charts/aiven-operator-crds/templates/aiven.io_serviceintegrations.yaml index 825f7270..5b48e66d 100644 --- a/charts/aiven-operator-crds/templates/aiven.io_serviceintegrations.yaml +++ b/charts/aiven-operator-crds/templates/aiven.io_serviceintegrations.yaml @@ -73,6 +73,9 @@ spec: - key - name type: object + autoscaler: + description: Autoscaler specific user configuration options + type: object clickhouseKafka: description: Clickhouse Kafka configuration values properties: diff --git a/config/crd/bases/aiven.io_serviceintegrationendpoints.yaml b/config/crd/bases/aiven.io_serviceintegrationendpoints.yaml index 2893db2a..679f0033 100644 --- a/config/crd/bases/aiven.io_serviceintegrationendpoints.yaml +++ b/config/crd/bases/aiven.io_serviceintegrationendpoints.yaml @@ -69,6 +69,35 @@ spec: - key - name type: object + autoscaler: + description: Autoscaler configuration values + properties: + autoscaling: + description: Configure autoscaling thresholds for a service + items: + description: AutoscalingProperties + properties: + cap_gb: + description: + The maximum total disk size (in gb) to allow + autoscaler to scale up to + maximum: 10000 + minimum: 50 + type: integer + type: + description: Type of autoscale event + enum: + - autoscale_disk + type: string + required: + - cap_gb + - type + type: object + maxItems: 64 + type: array + required: + - autoscaling + type: object datadog: description: Datadog configuration values properties: diff --git a/config/crd/bases/aiven.io_serviceintegrations.yaml b/config/crd/bases/aiven.io_serviceintegrations.yaml index 825f7270..5b48e66d 100644 --- a/config/crd/bases/aiven.io_serviceintegrations.yaml +++ b/config/crd/bases/aiven.io_serviceintegrations.yaml @@ -73,6 +73,9 @@ spec: - key - name type: object + autoscaler: + description: Autoscaler specific user configuration options + type: object clickhouseKafka: description: Clickhouse Kafka configuration values properties: diff --git a/docs/docs/api-reference/serviceintegration.md b/docs/docs/api-reference/serviceintegration.md index 1a0ed1cf..8f37c469 100644 --- a/docs/docs/api-reference/serviceintegration.md +++ b/docs/docs/api-reference/serviceintegration.md @@ -254,6 +254,7 @@ ServiceIntegrationSpec defines the desired state of ServiceIntegration. **Optional** - [`authSecretRef`](#spec.authSecretRef-property){: name='spec.authSecretRef-property'} (object). Authentication reference to Aiven token in a secret. See below for [nested schema](#spec.authSecretRef). +- [`autoscaler`](#spec.autoscaler-property){: name='spec.autoscaler-property'} (object). Autoscaler specific user configuration options. - [`clickhouseKafka`](#spec.clickhouseKafka-property){: name='spec.clickhouseKafka-property'} (object). Clickhouse Kafka configuration values. See below for [nested schema](#spec.clickhouseKafka). - [`clickhousePostgresql`](#spec.clickhousePostgresql-property){: name='spec.clickhousePostgresql-property'} (object). Clickhouse PostgreSQL configuration values. See below for [nested schema](#spec.clickhousePostgresql). - [`datadog`](#spec.datadog-property){: name='spec.datadog-property'} (object). Datadog specific user configuration options. See below for [nested schema](#spec.datadog). diff --git a/docs/docs/api-reference/serviceintegrationendpoint.md b/docs/docs/api-reference/serviceintegrationendpoint.md index ce48b8a7..10144680 100644 --- a/docs/docs/api-reference/serviceintegrationendpoint.md +++ b/docs/docs/api-reference/serviceintegrationendpoint.md @@ -95,6 +95,7 @@ ServiceIntegrationEndpointSpec defines the desired state of ServiceIntegrationEn **Optional** - [`authSecretRef`](#spec.authSecretRef-property){: name='spec.authSecretRef-property'} (object). Authentication reference to Aiven token in a secret. See below for [nested schema](#spec.authSecretRef). +- [`autoscaler`](#spec.autoscaler-property){: name='spec.autoscaler-property'} (object). Autoscaler configuration values. See below for [nested schema](#spec.autoscaler). - [`datadog`](#spec.datadog-property){: name='spec.datadog-property'} (object). Datadog configuration values. See below for [nested schema](#spec.datadog). - [`endpointName`](#spec.endpointName-property){: name='spec.endpointName-property'} (string, Immutable, MaxLength: 36). Source endpoint for the integration (if any). - [`externalAWSCloudwatchLogs`](#spec.externalAWSCloudwatchLogs-property){: name='spec.externalAWSCloudwatchLogs-property'} (object). ExternalAwsCloudwatchLogs configuration values. See below for [nested schema](#spec.externalAWSCloudwatchLogs). @@ -121,6 +122,27 @@ Authentication reference to Aiven token in a secret. - [`key`](#spec.authSecretRef.key-property){: name='spec.authSecretRef.key-property'} (string, MinLength: 1). - [`name`](#spec.authSecretRef.name-property){: name='spec.authSecretRef.name-property'} (string, MinLength: 1). +## autoscaler {: #spec.autoscaler } + +_Appears on [`spec`](#spec)._ + +Autoscaler configuration values. + +**Required** + +- [`autoscaling`](#spec.autoscaler.autoscaling-property){: name='spec.autoscaler.autoscaling-property'} (array of objects, MaxItems: 64). Configure autoscaling thresholds for a service. See below for [nested schema](#spec.autoscaler.autoscaling). + +### autoscaling {: #spec.autoscaler.autoscaling } + +_Appears on [`spec.autoscaler`](#spec.autoscaler)._ + +AutoscalingProperties. + +**Required** + +- [`cap_gb`](#spec.autoscaler.autoscaling.cap_gb-property){: name='spec.autoscaler.autoscaling.cap_gb-property'} (integer, Minimum: 50, Maximum: 10000). The maximum total disk size (in gb) to allow autoscaler to scale up to. +- [`type`](#spec.autoscaler.autoscaling.type-property){: name='spec.autoscaler.autoscaling.type-property'} (string, Enum: `autoscale_disk`). Type of autoscale event. + ## datadog {: #spec.datadog } _Appears on [`spec`](#spec)._ From de975aa80dd0a2176b6012b806c2d2fd395a1cb3 Mon Sep 17 00:00:00 2001 From: Timo Riski Date: Tue, 29 Oct 2024 16:27:46 +0200 Subject: [PATCH 4/5] feat: add tests for autoscaler integration --- .../serviceintegration.autoscaler.yaml | 29 ++++++++++ ...serviceintegrationendpoint.autoscaler.yaml | 17 ++++++ docs/docs/api-reference/serviceintegration.md | 57 +++++++++++++++++- .../serviceintegrationendpoint.md | 25 +++++++- tests/serviceintegration_test.go | 58 +++++++++++++++++++ tests/serviceintegrationendpoint_test.go | 42 ++++++++++++++ 6 files changed, 224 insertions(+), 4 deletions(-) create mode 100644 docs/docs/api-reference/examples/serviceintegration.autoscaler.yaml create mode 100644 docs/docs/api-reference/examples/serviceintegrationendpoint.autoscaler.yaml diff --git a/docs/docs/api-reference/examples/serviceintegration.autoscaler.yaml b/docs/docs/api-reference/examples/serviceintegration.autoscaler.yaml new file mode 100644 index 00000000..dd9c4ba5 --- /dev/null +++ b/docs/docs/api-reference/examples/serviceintegration.autoscaler.yaml @@ -0,0 +1,29 @@ +apiVersion: aiven.io/v1alpha1 +kind: ServiceIntegration +metadata: + name: my-service-integration +spec: + authSecretRef: + name: aiven-token + key: token + + project: aiven-project-name + integrationType: autoscaler + sourceServiceName: my-pg + # Look up autoscaler integration endpoint ID via Console + destinationEndpointId: my-destination-endpoint-id + +--- + +apiVersion: aiven.io/v1alpha1 +kind: PostgreSQL +metadata: + name: my-pg +spec: + authSecretRef: + name: aiven-token + key: token + + project: aiven-project-name + cloudName: google-europe-west1 + plan: startup-4 diff --git a/docs/docs/api-reference/examples/serviceintegrationendpoint.autoscaler.yaml b/docs/docs/api-reference/examples/serviceintegrationendpoint.autoscaler.yaml new file mode 100644 index 00000000..299d1b1a --- /dev/null +++ b/docs/docs/api-reference/examples/serviceintegrationendpoint.autoscaler.yaml @@ -0,0 +1,17 @@ +apiVersion: aiven.io/v1alpha1 +kind: ServiceIntegrationEndpoint +metadata: + name: my-service-integration-endpoint +spec: + authSecretRef: + name: aiven-token + key: token + + project: aiven-project-name + endpointName: my-autoscaler + endpointType: autoscaler + + autoscaler: + autoscaling: + - type: autoscale_disk + cap_gb: 100 diff --git a/docs/docs/api-reference/serviceintegration.md b/docs/docs/api-reference/serviceintegration.md index 8f37c469..d2bea8f8 100644 --- a/docs/docs/api-reference/serviceintegration.md +++ b/docs/docs/api-reference/serviceintegration.md @@ -4,6 +4,59 @@ title: "ServiceIntegration" ## Usage examples +??? example "autoscaler" + ```yaml + apiVersion: aiven.io/v1alpha1 + kind: ServiceIntegration + metadata: + name: my-service-integration + spec: + authSecretRef: + name: aiven-token + key: token + + project: aiven-project-name + integrationType: autoscaler + sourceServiceName: my-pg + # Specify autoscaler integration endpoint ID + destinationEndpointId: e8417f8c-513f-487a-9213-5a903a8e62d9 + + --- + + apiVersion: aiven.io/v1alpha1 + kind: ServiceIntegrationEndpoint + metadata: + name: my-service-integration-endpoint + spec: + authSecretRef: + name: aiven-token + key: token + + project: aiven-project-name + endpointName: my-autoscaler + endpointType: autoscaler + + autoscaler: + autoscaling: + - type: autoscale_disk + cap_gb: 100 + + --- + + apiVersion: aiven.io/v1alpha1 + kind: PostgreSQL + metadata: + name: my-pg + spec: + authSecretRef: + name: aiven-token + key: token + + project: aiven-project-name + cloudName: google-europe-west1 + plan: startup-4 + ``` + ??? example "clickhouse_postgresql" ```yaml apiVersion: aiven.io/v1alpha1 @@ -225,8 +278,8 @@ kubectl get serviceintegrations my-service-integration The output is similar to the following: ```shell -Name Project Type Source Service Name Destination Service Name -my-service-integration aiven-project-name clickhouse_postgresql my-pg my-clickhouse +Name Project Type Source Service Name Destination Endpoint ID +my-service-integration aiven-project-name autoscaler my-pg e8417f8c-513f-487a-9213-5a903a8e62d9 ``` ## ServiceIntegration {: #ServiceIntegration } diff --git a/docs/docs/api-reference/serviceintegrationendpoint.md b/docs/docs/api-reference/serviceintegrationendpoint.md index 10144680..8e048fc2 100644 --- a/docs/docs/api-reference/serviceintegrationendpoint.md +++ b/docs/docs/api-reference/serviceintegrationendpoint.md @@ -4,6 +4,27 @@ title: "ServiceIntegrationEndpoint" ## Usage examples +??? example "autoscaler" + ```yaml + apiVersion: aiven.io/v1alpha1 + kind: ServiceIntegrationEndpoint + metadata: + name: my-service-integration-endpoint + spec: + authSecretRef: + name: aiven-token + key: token + + project: aiven-project-name + endpointName: my-autoscaler + endpointType: autoscaler + + autoscaler: + autoscaling: + - type: autoscale_disk + cap_gb: 100 + ``` + ??? example "external_postgresql" ```yaml apiVersion: aiven.io/v1alpha1 @@ -66,8 +87,8 @@ kubectl get serviceintegrationendpoints my-service-integration-endpoint The output is similar to the following: ```shell -Name Project Endpoint Name Endpoint Type ID -my-service-integration-endpoint aiven-project-name my-external-postgresql external_postgresql +Name Project Endpoint Name Endpoint Type ID +my-service-integration-endpoint aiven-project-name my-autoscaler autoscaler ``` ## ServiceIntegrationEndpoint {: #ServiceIntegrationEndpoint } diff --git a/tests/serviceintegration_test.go b/tests/serviceintegration_test.go index ee164f74..cc6ed15c 100644 --- a/tests/serviceintegration_test.go +++ b/tests/serviceintegration_test.go @@ -225,6 +225,64 @@ func TestServiceIntegrationKafkaConnect(t *testing.T) { assert.Equal(t, "__connect_offsets", *si.Spec.KafkaConnectUserConfig.KafkaConnect.OffsetStorageTopic) } +func TestServiceIntegrationAutoscaler(t *testing.T) { + t.Parallel() + defer recoverPanic(t) + + endpointID := os.Getenv("AUTOSCALER_ENDPOINT_ID") + if endpointID == "" { + t.Skip("Provide AUTOSCALER_ENDPOINT_ID for this test") + } + + // GIVEN + ctx, cancel := testCtx() + defer cancel() + + pgName := randName("postgresql") + siName := randName("autoscaler") + + yml, err := loadExampleYaml("serviceintegration.autoscaler.yaml", map[string]string{ + "aiven-project-name": cfg.Project, + "google-europe-west1": cfg.PrimaryCloudName, + "my-pg": pgName, + "my-service-integration": siName, + "my-destination-endpoint-id": endpointID, + }) + require.NoError(t, err) + s := NewSession(ctx, k8sClient, cfg.Project) + + // Cleans test afterward + defer s.Destroy(t) + + // WHEN + // Applies given manifest + require.NoError(t, s.Apply(yml)) + + // Waits kube objects + pg := new(v1alpha1.PostgreSQL) + require.NoError(t, s.GetRunning(pg, pgName)) + + si := new(v1alpha1.ServiceIntegration) + require.NoError(t, s.GetRunning(si, siName)) + + // THEN + // Validates PostgreSQL + pgAvn, err := avnGen.ServiceGet(ctx, cfg.Project, pgName) + require.NoError(t, err) + assert.Equal(t, pgAvn.ServiceName, pg.GetName()) + assert.Contains(t, serviceRunningStatesAiven, pgAvn.State) + + // Validates ServiceIntegration + siAvn, err := avnGen.ServiceIntegrationGet(ctx, cfg.Project, si.Status.ID) + require.NoError(t, err) + assert.EqualValues(t, "autoscaler", siAvn.IntegrationType) + assert.EqualValues(t, siAvn.IntegrationType, si.Spec.IntegrationType) + assert.Equal(t, pgName, siAvn.SourceService) + assert.Equal(t, endpointID, *siAvn.DestEndpointId) + assert.True(t, siAvn.Active) + assert.True(t, siAvn.Enabled) +} + // todo: refactor when ServiceIntegrationEndpoint released func TestServiceIntegrationDatadog(t *testing.T) { t.Parallel() diff --git a/tests/serviceintegrationendpoint_test.go b/tests/serviceintegrationendpoint_test.go index 1fa0dfdb..59a4f87f 100644 --- a/tests/serviceintegrationendpoint_test.go +++ b/tests/serviceintegrationendpoint_test.go @@ -95,3 +95,45 @@ func TestServiceIntegrationEndpoint(t *testing.T) { assert.EqualValues(t, endpointRegistryAvn.UserConfig["basic_auth_username"], *endpointRegistry.Spec.ExternalSchemaRegistry.BasicAuthUsername) assert.EqualValues(t, endpointRegistryAvn.UserConfig["basic_auth_password"], *endpointRegistry.Spec.ExternalSchemaRegistry.BasicAuthPassword) } + +func TestServiceIntegrationEndpointAutoscaler(t *testing.T) { + t.Parallel() + defer recoverPanic(t) + + // GIVEN + ctx, cancel := testCtx() + defer cancel() + + endpointName := randName("autoscaler") + + yml, err := loadExampleYaml("serviceintegrationendpoint.autoscaler.yaml", map[string]string{ + "aiven-project-name": cfg.Project, + "my-service-integration-endpoint": endpointName, + }) + require.NoError(t, err) + s := NewSession(ctx, k8sClient, cfg.Project) + + // Cleans test afterward + defer s.Destroy(t) + + // WHEN + // Applies given manifest + require.NoError(t, s.Apply(yml)) + + // THEN + + // Validates autoscaler ServiceIntegrationEndpoint + endpointAutoscaler := new(v1alpha1.ServiceIntegrationEndpoint) + require.NoError(t, s.GetRunning(endpointAutoscaler, endpointName)) + + endpointAvn, err := avnGen.ServiceIntegrationEndpointGet(ctx, cfg.Project, endpointAutoscaler.Status.ID, service.ServiceIntegrationEndpointGetIncludeSecrets(true)) + require.NoError(t, err) + + assert.EqualValues(t, "autoscaler", endpointAvn.EndpointType) + assert.EqualValues(t, endpointAvn.EndpointType, endpointAutoscaler.Spec.EndpointType) + // TODO: remove type assertions once generated client has full user config typing + assert.EqualValues(t, "autoscale_disk", endpointAutoscaler.Spec.Autoscaler.Autoscaling[0].Type) + assert.EqualValues(t, "autoscale_disk", endpointAvn.UserConfig["autoscaling"].([]interface{})[0].(map[string]interface{})["type"]) + assert.EqualValues(t, 100, endpointAutoscaler.Spec.Autoscaler.Autoscaling[0].CapGb) + assert.EqualValues(t, 100, endpointAvn.UserConfig["autoscaling"].([]interface{})[0].(map[string]interface{})["cap_gb"]) +} From 0bc729f4aae3489c5c01e526afb6658eeeb89441 Mon Sep 17 00:00:00 2001 From: Timo Riski Date: Wed, 6 Nov 2024 14:23:38 +0200 Subject: [PATCH 5/5] feat: add validation for disk space / autoscaler integration --- controllers/generic_service_handler.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/controllers/generic_service_handler.go b/controllers/generic_service_handler.go index 4d6093fa..55256a46 100644 --- a/controllers/generic_service_handler.go +++ b/controllers/generic_service_handler.go @@ -55,6 +55,15 @@ func (h *genericServiceHandler) createOrUpdate(ctx context.Context, avn *aiven.C technicalEmails = append(technicalEmails, aiven.ContactEmail(email)) } + diskSpace := v1alpha1.ConvertDiskSpace(o.getDiskSpace()) + if diskSpace > 0 { + for _, v := range oldService.ServiceIntegrations { + if v.IntegrationType == service.IntegrationTypeAutoscaler { + return fmt.Errorf("cannot set disk space for service with autoscaler integration enabled") + } + } + } + // Creates if not exists or updates existing service var reason string if !exists { @@ -66,7 +75,7 @@ func (h *genericServiceHandler) createOrUpdate(ctx context.Context, avn *aiven.C req := aiven.CreateServiceRequest{ Cloud: spec.CloudName, - DiskSpaceMB: v1alpha1.ConvertDiskSpace(o.getDiskSpace()), + DiskSpaceMB: diskSpace, MaintenanceWindow: getMaintenanceWindow(spec.MaintenanceWindowDow, spec.MaintenanceWindowTime), Plan: spec.Plan, ProjectVPCID: toOptionalStringPointer(projectVPCID), @@ -105,7 +114,7 @@ func (h *genericServiceHandler) createOrUpdate(ctx context.Context, avn *aiven.C req := aiven.UpdateServiceRequest{ Cloud: spec.CloudName, - DiskSpaceMB: v1alpha1.ConvertDiskSpace(o.getDiskSpace()), + DiskSpaceMB: diskSpace, MaintenanceWindow: getMaintenanceWindow(spec.MaintenanceWindowDow, spec.MaintenanceWindowTime), Plan: spec.Plan, Powered: true,