diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cca0645..12740713 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,35 @@ ## [MAJOR.MINOR.PATCH] - YYYY-MM-DD - Add kind: `ClickhouseGrant` +- Add `KafkaConnect` field `userConfig.secret_providers`, type `array`: Configure external secret providers + in order to reference external secrets in connector configuration +- Add `Kafka` field `userConfig.kafka_connect_secret_providers`, type `array`: Configure external secret + providers in order to reference external secrets in connector configuration +- Add `Kafka` field `userConfig.letsencrypt_sasl_privatelink`, type `boolean`: Use Letsencrypt CA for + Kafka SASL via Privatelink +- Add `ServiceIntegration` field `datadog.mirrormaker_custom_metrics`, type `array`: List of custom metrics +- Add `ServiceIntegration` field `kafkaMirrormaker.kafka_mirrormaker.consumer_auto_offset_reset`, type + `string`: Set where consumer starts to consume data +- Add `ServiceIntegration` field `kafkaMirrormaker.kafka_mirrormaker.consumer_max_poll_records`, type + `integer`: Set consumer max.poll.records. The default is 500 +- Change `PostgreSQL` field `userConfig.pgaudit.feature_enabled`: deprecated +- Change `PostgreSQL` field `userConfig.pgaudit.log_catalog`: deprecated +- Change `PostgreSQL` field `userConfig.pgaudit.log_client`: deprecated +- Change `PostgreSQL` field `userConfig.pgaudit.log_level`: deprecated, enum ~~`[debug1, debug2, debug3, +debug4, debug5, info, log, notice, warning]`~~ +- Change `PostgreSQL` field `userConfig.pgaudit.log_max_string_length`: deprecated +- Change `PostgreSQL` field `userConfig.pgaudit.log_nested_statements`: deprecated +- Change `PostgreSQL` field `userConfig.pgaudit.log_parameter_max_size`: deprecated +- Change `PostgreSQL` field `userConfig.pgaudit.log_parameter`: deprecated +- Change `PostgreSQL` field `userConfig.pgaudit.log_relation`: deprecated +- Change `PostgreSQL` field `userConfig.pgaudit.log_rows`: deprecated +- Change `PostgreSQL` field `userConfig.pgaudit.log_statement_once`: deprecated +- Change `PostgreSQL` field `userConfig.pgaudit.log_statement`: deprecated +- Change `PostgreSQL` field `userConfig.pgaudit.log`: deprecated +- Change `PostgreSQL` field `userConfig.pgaudit.role`: deprecated +- Change `PostgreSQL` field `userConfig.pgaudit`: deprecated +- Change `ServiceIntegrationEndpoint` field `externalPostgresql.ssl_mode`: enum ~~`[allow, disable, prefer, +require, verify-ca, verify-full]`~~ → `[require, verify-ca, verify-full]` ## v0.20.0 - 2024-06-05 diff --git a/api/v1alpha1/userconfig/integration/clickhouse_kafka/clickhouse_kafka.go b/api/v1alpha1/userconfig/integration/clickhouse_kafka/clickhouse_kafka.go index a75469e6..a7fc1dc5 100644 --- a/api/v1alpha1/userconfig/integration/clickhouse_kafka/clickhouse_kafka.go +++ b/api/v1alpha1/userconfig/integration/clickhouse_kafka/clickhouse_kafka.go @@ -34,7 +34,7 @@ type Tables struct { // Table columns Columns []*Columns `groups:"create,update" json:"columns"` - // +kubebuilder:validation:Enum="Avro";"CSV";"JSONAsString";"JSONCompactEachRow";"JSONCompactStringsEachRow";"JSONEachRow";"JSONStringsEachRow";"MsgPack";"TSKV";"TSV";"TabSeparated";"RawBLOB";"AvroConfluent" + // +kubebuilder:validation:Enum="Avro";"CSV";"JSONAsString";"JSONCompactEachRow";"JSONCompactStringsEachRow";"JSONEachRow";"JSONStringsEachRow";"MsgPack";"TSKV";"TSV";"TabSeparated";"RawBLOB";"AvroConfluent";"Parquet" // Message data format DataFormat string `groups:"create,update" json:"data_format"` diff --git a/api/v1alpha1/userconfig/integration/datadog/datadog.go b/api/v1alpha1/userconfig/integration/datadog/datadog.go index 29501d97..00106288 100644 --- a/api/v1alpha1/userconfig/integration/datadog/datadog.go +++ b/api/v1alpha1/userconfig/integration/datadog/datadog.go @@ -71,6 +71,10 @@ type DatadogUserConfig struct { // Maximum number of JMX metrics to send MaxJmxMetrics *int `groups:"create,update" json:"max_jmx_metrics,omitempty"` + // +kubebuilder:validation:MaxItems=1024 + // List of custom metrics + MirrormakerCustomMetrics []string `groups:"create,update" json:"mirrormaker_custom_metrics,omitempty"` + // Datadog Opensearch Options Opensearch *Opensearch `groups:"create,update" json:"opensearch,omitempty"` diff --git a/api/v1alpha1/userconfig/integration/datadog/zz_generated.deepcopy.go b/api/v1alpha1/userconfig/integration/datadog/zz_generated.deepcopy.go index 869f4b3a..a3c9cb0e 100644 --- a/api/v1alpha1/userconfig/integration/datadog/zz_generated.deepcopy.go +++ b/api/v1alpha1/userconfig/integration/datadog/zz_generated.deepcopy.go @@ -80,6 +80,11 @@ func (in *DatadogUserConfig) DeepCopyInto(out *DatadogUserConfig) { *out = new(int) **out = **in } + if in.MirrormakerCustomMetrics != nil { + in, out := &in.MirrormakerCustomMetrics, &out.MirrormakerCustomMetrics + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.Opensearch != nil { in, out := &in.Opensearch, &out.Opensearch *out = new(Opensearch) diff --git a/api/v1alpha1/userconfig/integration/kafka_mirrormaker/kafka_mirrormaker.go b/api/v1alpha1/userconfig/integration/kafka_mirrormaker/kafka_mirrormaker.go index b03e4cc0..b091f00d 100644 --- a/api/v1alpha1/userconfig/integration/kafka_mirrormaker/kafka_mirrormaker.go +++ b/api/v1alpha1/userconfig/integration/kafka_mirrormaker/kafka_mirrormaker.go @@ -5,11 +5,20 @@ package kafkamirrormakeruserconfig // Kafka MirrorMaker configuration values type KafkaMirrormaker struct { + // +kubebuilder:validation:Enum="earliest";"latest" + // Set where consumer starts to consume data. Value `earliest`: Start replication from the earliest offset. Value `latest`: Start replication from the latest offset. Default is `earliest`. + ConsumerAutoOffsetReset *string `groups:"create,update" json:"consumer_auto_offset_reset,omitempty"` + // +kubebuilder:validation:Minimum=1 // +kubebuilder:validation:Maximum=5242880 // The minimum amount of data the server should return for a fetch request ConsumerFetchMinBytes *int `groups:"create,update" json:"consumer_fetch_min_bytes,omitempty"` + // +kubebuilder:validation:Minimum=100 + // +kubebuilder:validation:Maximum=20000 + // Set consumer max.poll.records. The default is 500. + ConsumerMaxPollRecords *int `groups:"create,update" json:"consumer_max_poll_records,omitempty"` + // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=5242880 // The batch size in bytes producer will attempt to collect before publishing to broker. diff --git a/api/v1alpha1/userconfig/integration/kafka_mirrormaker/zz_generated.deepcopy.go b/api/v1alpha1/userconfig/integration/kafka_mirrormaker/zz_generated.deepcopy.go index 6c31b8dd..19dc8146 100644 --- a/api/v1alpha1/userconfig/integration/kafka_mirrormaker/zz_generated.deepcopy.go +++ b/api/v1alpha1/userconfig/integration/kafka_mirrormaker/zz_generated.deepcopy.go @@ -9,11 +9,21 @@ package kafkamirrormakeruserconfig // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KafkaMirrormaker) DeepCopyInto(out *KafkaMirrormaker) { *out = *in + if in.ConsumerAutoOffsetReset != nil { + in, out := &in.ConsumerAutoOffsetReset, &out.ConsumerAutoOffsetReset + *out = new(string) + **out = **in + } if in.ConsumerFetchMinBytes != nil { in, out := &in.ConsumerFetchMinBytes, &out.ConsumerFetchMinBytes *out = new(int) **out = **in } + if in.ConsumerMaxPollRecords != nil { + in, out := &in.ConsumerMaxPollRecords, &out.ConsumerMaxPollRecords + *out = new(int) + **out = **in + } if in.ProducerBatchSize != nil { in, out := &in.ProducerBatchSize, &out.ProducerBatchSize *out = new(int) diff --git a/api/v1alpha1/userconfig/integrationendpoints/external_postgresql/external_postgresql.go b/api/v1alpha1/userconfig/integrationendpoints/external_postgresql/external_postgresql.go index 0f8aaed0..065995d8 100644 --- a/api/v1alpha1/userconfig/integrationendpoints/external_postgresql/external_postgresql.go +++ b/api/v1alpha1/userconfig/integrationendpoints/external_postgresql/external_postgresql.go @@ -30,8 +30,8 @@ type ExternalPostgresqlUserConfig struct { // Client key SslClientKey *string `groups:"create,update" json:"ssl_client_key,omitempty"` - // +kubebuilder:validation:Enum="disable";"allow";"prefer";"require";"verify-ca";"verify-full" - // SSL Mode + // +kubebuilder:validation:Enum="require";"verify-ca";"verify-full" + // SSL mode to use for the connection. Please note that Aiven requires TLS for all connections to external PostgreSQL services. SslMode *string `groups:"create,update" json:"ssl_mode,omitempty"` // +kubebuilder:validation:MaxLength=16384 diff --git a/api/v1alpha1/userconfig/service/kafka/kafka.go b/api/v1alpha1/userconfig/service/kafka/kafka.go index cffcb9a6..056b4c20 100644 --- a/api/v1alpha1/userconfig/service/kafka/kafka.go +++ b/api/v1alpha1/userconfig/service/kafka/kafka.go @@ -312,6 +312,57 @@ type KafkaConnectConfig struct { SessionTimeoutMs *int `groups:"create,update" json:"session_timeout_ms,omitempty"` } +// AWS config for Secret Provider +type Aws struct { + // +kubebuilder:validation:MaxLength=128 + // Access key used to authenticate with aws + AccessKey *string `groups:"create,update" json:"access_key,omitempty"` + + // +kubebuilder:validation:Enum="credentials" + // Auth method of the vault secret provider + AuthMethod string `groups:"create,update" json:"auth_method"` + + // +kubebuilder:validation:MaxLength=64 + // Region used to lookup secrets with AWS SecretManager + Region string `groups:"create,update" json:"region"` + + // +kubebuilder:validation:MaxLength=128 + // Secret key used to authenticate with aws + SecretKey *string `groups:"create,update" json:"secret_key,omitempty"` +} + +// Vault Config for Secret Provider +type Vault struct { + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=65536 + // Address of the Vault server + Address string `groups:"create,update" json:"address"` + + // +kubebuilder:validation:Enum="token" + // Auth method of the vault secret provider + AuthMethod string `groups:"create,update" json:"auth_method"` + + // +kubebuilder:validation:Enum=1;2 + // KV Secrets Engine version of the Vault server instance + EngineVersion *int `groups:"create,update" json:"engine_version,omitempty"` + + // +kubebuilder:validation:MaxLength=256 + // Token used to authenticate with vault and auth method `token`. + Token *string `groups:"create,update" json:"token,omitempty"` +} + +// SecretProvider +type KafkaConnectSecretProviders struct { + // AWS config for Secret Provider + Aws *Aws `groups:"create,update" json:"aws,omitempty"` + + // Name of the secret provider. Used to reference secrets in connector config. + Name string `groups:"create,update" json:"name"` + + // Vault Config for Secret Provider + Vault *Vault `groups:"create,update" json:"vault,omitempty"` +} + // Kafka REST configuration type KafkaRestConfig struct { // If true the consumer's offset will be periodically committed to Kafka in the background @@ -474,6 +525,9 @@ type KafkaUserConfig struct { // Kafka Connect configuration values KafkaConnectConfig *KafkaConnectConfig `groups:"create,update" json:"kafka_connect_config,omitempty"` + // Configure external secret providers in order to reference external secrets in connector configuration. Currently Hashicorp Vault (provider: vault, auth_method: token) and AWS Secrets Manager (provider: aws, auth_method: credentials) are supported. Secrets can be referenced in connector config with ${::} + KafkaConnectSecretProviders []*KafkaConnectSecretProviders `groups:"create,update" json:"kafka_connect_secret_providers,omitempty"` + // Enable Kafka-REST service KafkaRest *bool `groups:"create,update" json:"kafka_rest,omitempty"` @@ -487,6 +541,9 @@ type KafkaUserConfig struct { // Kafka major version KafkaVersion *string `groups:"create,update" json:"kafka_version,omitempty"` + // Use Letsencrypt CA for Kafka SASL via Privatelink + LetsencryptSaslPrivatelink *bool `groups:"create,update" json:"letsencrypt_sasl_privatelink,omitempty"` + // Allow access to selected service ports from private networks PrivateAccess *PrivateAccess `groups:"create,update" json:"private_access,omitempty"` diff --git a/api/v1alpha1/userconfig/service/kafka/zz_generated.deepcopy.go b/api/v1alpha1/userconfig/service/kafka/zz_generated.deepcopy.go index 909e79b3..6e1772d4 100644 --- a/api/v1alpha1/userconfig/service/kafka/zz_generated.deepcopy.go +++ b/api/v1alpha1/userconfig/service/kafka/zz_generated.deepcopy.go @@ -6,6 +6,31 @@ package kafkauserconfig +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Aws) DeepCopyInto(out *Aws) { + *out = *in + if in.AccessKey != nil { + in, out := &in.AccessKey, &out.AccessKey + *out = new(string) + **out = **in + } + if in.SecretKey != nil { + in, out := &in.SecretKey, &out.SecretKey + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Aws. +func (in *Aws) DeepCopy() *Aws { + if in == nil { + return nil + } + out := new(Aws) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IpFilter) DeepCopyInto(out *IpFilter) { *out = *in @@ -391,6 +416,31 @@ func (in *KafkaConnectConfig) DeepCopy() *KafkaConnectConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KafkaConnectSecretProviders) DeepCopyInto(out *KafkaConnectSecretProviders) { + *out = *in + if in.Aws != nil { + in, out := &in.Aws, &out.Aws + *out = new(Aws) + (*in).DeepCopyInto(*out) + } + if in.Vault != nil { + in, out := &in.Vault, &out.Vault + *out = new(Vault) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KafkaConnectSecretProviders. +func (in *KafkaConnectSecretProviders) DeepCopy() *KafkaConnectSecretProviders { + if in == nil { + return nil + } + out := new(KafkaConnectSecretProviders) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KafkaRestConfig) DeepCopyInto(out *KafkaRestConfig) { *out = *in @@ -505,6 +555,17 @@ func (in *KafkaUserConfig) DeepCopyInto(out *KafkaUserConfig) { *out = new(KafkaConnectConfig) (*in).DeepCopyInto(*out) } + if in.KafkaConnectSecretProviders != nil { + in, out := &in.KafkaConnectSecretProviders, &out.KafkaConnectSecretProviders + *out = make([]*KafkaConnectSecretProviders, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(KafkaConnectSecretProviders) + (*in).DeepCopyInto(*out) + } + } + } if in.KafkaRest != nil { in, out := &in.KafkaRest, &out.KafkaRest *out = new(bool) @@ -525,6 +586,11 @@ func (in *KafkaUserConfig) DeepCopyInto(out *KafkaUserConfig) { *out = new(string) **out = **in } + if in.LetsencryptSaslPrivatelink != nil { + in, out := &in.LetsencryptSaslPrivatelink, &out.LetsencryptSaslPrivatelink + *out = new(bool) + **out = **in + } if in.PrivateAccess != nil { in, out := &in.PrivateAccess, &out.PrivateAccess *out = new(PrivateAccess) @@ -771,3 +837,28 @@ func (in *TieredStorage) DeepCopy() *TieredStorage { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Vault) DeepCopyInto(out *Vault) { + *out = *in + if in.EngineVersion != nil { + in, out := &in.EngineVersion, &out.EngineVersion + *out = new(int) + **out = **in + } + if in.Token != nil { + in, out := &in.Token, &out.Token + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Vault. +func (in *Vault) DeepCopy() *Vault { + if in == nil { + return nil + } + out := new(Vault) + in.DeepCopyInto(out) + return out +} diff --git a/api/v1alpha1/userconfig/service/kafka_connect/kafka_connect.go b/api/v1alpha1/userconfig/service/kafka_connect/kafka_connect.go index d559cc78..2fbf3a24 100644 --- a/api/v1alpha1/userconfig/service/kafka_connect/kafka_connect.go +++ b/api/v1alpha1/userconfig/service/kafka_connect/kafka_connect.go @@ -122,6 +122,57 @@ type PublicAccess struct { // Allow clients to connect to prometheus from the public internet for service nodes that are in a project VPC or another type of private network Prometheus *bool `groups:"create,update" json:"prometheus,omitempty"` } + +// AWS config for Secret Provider +type Aws struct { + // +kubebuilder:validation:MaxLength=128 + // Access key used to authenticate with aws + AccessKey *string `groups:"create,update" json:"access_key,omitempty"` + + // +kubebuilder:validation:Enum="credentials" + // Auth method of the vault secret provider + AuthMethod string `groups:"create,update" json:"auth_method"` + + // +kubebuilder:validation:MaxLength=64 + // Region used to lookup secrets with AWS SecretManager + Region string `groups:"create,update" json:"region"` + + // +kubebuilder:validation:MaxLength=128 + // Secret key used to authenticate with aws + SecretKey *string `groups:"create,update" json:"secret_key,omitempty"` +} + +// Vault Config for Secret Provider +type Vault struct { + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=65536 + // Address of the Vault server + Address string `groups:"create,update" json:"address"` + + // +kubebuilder:validation:Enum="token" + // Auth method of the vault secret provider + AuthMethod string `groups:"create,update" json:"auth_method"` + + // +kubebuilder:validation:Enum=1;2 + // KV Secrets Engine version of the Vault server instance + EngineVersion *int `groups:"create,update" json:"engine_version,omitempty"` + + // +kubebuilder:validation:MaxLength=256 + // Token used to authenticate with vault and auth method `token`. + Token *string `groups:"create,update" json:"token,omitempty"` +} + +// SecretProvider +type SecretProviders struct { + // AWS config for Secret Provider + Aws *Aws `groups:"create,update" json:"aws,omitempty"` + + // Name of the secret provider. Used to reference secrets in connector config. + Name string `groups:"create,update" json:"name"` + + // Vault Config for Secret Provider + Vault *Vault `groups:"create,update" json:"vault,omitempty"` +} type KafkaConnectUserConfig struct { // +kubebuilder:validation:MaxItems=1 // +kubebuilder:deprecatedversion:warning="additional_backup_regions is deprecated" @@ -144,6 +195,9 @@ type KafkaConnectUserConfig struct { // Allow access to selected service ports from the public Internet PublicAccess *PublicAccess `groups:"create,update" json:"public_access,omitempty"` + // Configure external secret providers in order to reference external secrets in connector configuration. Currently Hashicorp Vault (provider: vault, auth_method: token) and AWS Secrets Manager (provider: aws, auth_method: credentials) are supported. Secrets can be referenced in connector config with ${::} + SecretProviders []*SecretProviders `groups:"create,update" json:"secret_providers,omitempty"` + // Store logs for the service so that they are available in the HTTP API and console. ServiceLog *bool `groups:"create,update" json:"service_log,omitempty"` diff --git a/api/v1alpha1/userconfig/service/kafka_connect/zz_generated.deepcopy.go b/api/v1alpha1/userconfig/service/kafka_connect/zz_generated.deepcopy.go index 4e0be9de..45a6cfa6 100644 --- a/api/v1alpha1/userconfig/service/kafka_connect/zz_generated.deepcopy.go +++ b/api/v1alpha1/userconfig/service/kafka_connect/zz_generated.deepcopy.go @@ -6,6 +6,31 @@ package kafkaconnectuserconfig +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Aws) DeepCopyInto(out *Aws) { + *out = *in + if in.AccessKey != nil { + in, out := &in.AccessKey, &out.AccessKey + *out = new(string) + **out = **in + } + if in.SecretKey != nil { + in, out := &in.SecretKey, &out.SecretKey + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Aws. +func (in *Aws) DeepCopy() *Aws { + if in == nil { + return nil + } + out := new(Aws) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IpFilter) DeepCopyInto(out *IpFilter) { *out = *in @@ -160,6 +185,17 @@ func (in *KafkaConnectUserConfig) DeepCopyInto(out *KafkaConnectUserConfig) { *out = new(PublicAccess) (*in).DeepCopyInto(*out) } + if in.SecretProviders != nil { + in, out := &in.SecretProviders, &out.SecretProviders + *out = make([]*SecretProviders, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(SecretProviders) + (*in).DeepCopyInto(*out) + } + } + } if in.ServiceLog != nil { in, out := &in.ServiceLog, &out.ServiceLog *out = new(bool) @@ -261,3 +297,53 @@ func (in *PublicAccess) DeepCopy() *PublicAccess { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretProviders) DeepCopyInto(out *SecretProviders) { + *out = *in + if in.Aws != nil { + in, out := &in.Aws, &out.Aws + *out = new(Aws) + (*in).DeepCopyInto(*out) + } + if in.Vault != nil { + in, out := &in.Vault, &out.Vault + *out = new(Vault) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretProviders. +func (in *SecretProviders) DeepCopy() *SecretProviders { + if in == nil { + return nil + } + out := new(SecretProviders) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Vault) DeepCopyInto(out *Vault) { + *out = *in + if in.EngineVersion != nil { + in, out := &in.EngineVersion, &out.EngineVersion + *out = new(int) + **out = **in + } + if in.Token != nil { + in, out := &in.Token, &out.Token + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Vault. +func (in *Vault) DeepCopy() *Vault { + if in == nil { + return nil + } + out := new(Vault) + in.DeepCopyInto(out) + return out +} diff --git a/api/v1alpha1/userconfig/service/pg/pg.go b/api/v1alpha1/userconfig/service/pg/pg.go index b2dd637a..d0198d86 100644 --- a/api/v1alpha1/userconfig/service/pg/pg.go +++ b/api/v1alpha1/userconfig/service/pg/pg.go @@ -143,7 +143,7 @@ type Pg struct { LogErrorVerbosity *string `groups:"create,update" json:"log_error_verbosity,omitempty"` // +kubebuilder:validation:Enum="'pid=%p,user=%u,db=%d,app=%a,client=%h '";"'%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h '";"'%m [%p] %q[user=%u,db=%d,app=%a] '" - // Choose from one of the available log-formats. These can support popular log analyzers like pgbadger, pganalyze etc. + // Choose from one of the available log formats. LogLinePrefix *string `groups:"create,update" json:"log_line_prefix,omitempty"` // +kubebuilder:validation:Minimum=-1 @@ -309,53 +309,66 @@ type PgQualstats struct { TrackPgCatalog *bool `groups:"create,update" json:"track_pg_catalog,omitempty"` } -// System-wide settings for the pgaudit extension +// Deprecated. System-wide settings for the pgaudit extension type Pgaudit struct { - // Enable pgaudit extension. When enabled, pgaudit extension will be automatically installed.Otherwise, extension will be uninstalled but auditing configurations will be preserved. + // +kubebuilder:deprecatedversion:warning="feature_enabled is deprecated" + // Deprecated. Enable pgaudit extension. When enabled, pgaudit extension will be automatically installed.Otherwise, extension will be uninstalled but auditing configurations will be preserved. FeatureEnabled *bool `groups:"create,update" json:"feature_enabled,omitempty"` - // Specifies which classes of statements will be logged by session audit logging. + // +kubebuilder:deprecatedversion:warning="log is deprecated" + // Deprecated. Specifies which classes of statements will be logged by session audit logging. Log []string `groups:"create,update" json:"log,omitempty"` - // Specifies that session logging should be enabled in the casewhere all relations in a statement are in pg_catalog. + // +kubebuilder:deprecatedversion:warning="log_catalog is deprecated" + // Deprecated. Specifies that session logging should be enabled in the casewhere all relations in a statement are in pg_catalog. LogCatalog *bool `groups:"create,update" json:"log_catalog,omitempty"` - // Specifies whether log messages will be visible to a client process such as psql. + // +kubebuilder:deprecatedversion:warning="log_client is deprecated" + // Deprecated. Specifies whether log messages will be visible to a client process such as psql. LogClient *bool `groups:"create,update" json:"log_client,omitempty"` - // +kubebuilder:validation:Enum="debug1";"debug2";"debug3";"debug4";"debug5";"info";"notice";"warning";"log" - // Specifies the log level that will be used for log entries. + // +kubebuilder:deprecatedversion:warning="log_level is deprecated" + // Deprecated. Specifies the log level that will be used for log entries. LogLevel *string `groups:"create,update" json:"log_level,omitempty"` // +kubebuilder:validation:Minimum=-1 // +kubebuilder:validation:Maximum=102400 - // Crop parameters representation and whole statements if they exceed this threshold. A (default) value of -1 disable the truncation. + // +kubebuilder:deprecatedversion:warning="log_max_string_length is deprecated" + // Deprecated. Crop parameters representation and whole statements if they exceed this threshold. A (default) value of -1 disable the truncation. LogMaxStringLength *int `groups:"create,update" json:"log_max_string_length,omitempty"` - // This GUC allows to turn off logging nested statements, that is, statements that are executed as part of another ExecutorRun. + // +kubebuilder:deprecatedversion:warning="log_nested_statements is deprecated" + // Deprecated. This GUC allows to turn off logging nested statements, that is, statements that are executed as part of another ExecutorRun. LogNestedStatements *bool `groups:"create,update" json:"log_nested_statements,omitempty"` - // Specifies that audit logging should include the parameters that were passed with the statement. + // +kubebuilder:deprecatedversion:warning="log_parameter is deprecated" + // Deprecated. Specifies that audit logging should include the parameters that were passed with the statement. LogParameter *bool `groups:"create,update" json:"log_parameter,omitempty"` - // Specifies that parameter values longer than this setting (in bytes) should not be logged, but replaced with . + // +kubebuilder:deprecatedversion:warning="log_parameter_max_size is deprecated" + // Deprecated. Specifies that parameter values longer than this setting (in bytes) should not be logged, but replaced with . LogParameterMaxSize *int `groups:"create,update" json:"log_parameter_max_size,omitempty"` - // Specifies whether session audit logging should create a separate log entry for each relation (TABLE, VIEW, etc.) referenced in a SELECT or DML statement. + // +kubebuilder:deprecatedversion:warning="log_relation is deprecated" + // Deprecated. Specifies whether session audit logging should create a separate log entry for each relation (TABLE, VIEW, etc.) referenced in a SELECT or DML statement. LogRelation *bool `groups:"create,update" json:"log_relation,omitempty"` - // Specifies that audit logging should include the rows retrieved or affected by a statement. When enabled the rows field will be included after the parameter field. + // +kubebuilder:deprecatedversion:warning="log_rows is deprecated" + // Deprecated. Specifies that audit logging should include the rows retrieved or affected by a statement. When enabled the rows field will be included after the parameter field. LogRows *bool `groups:"create,update" json:"log_rows,omitempty"` - // Specifies whether logging will include the statement text and parameters (if enabled). + // +kubebuilder:deprecatedversion:warning="log_statement is deprecated" + // Deprecated. Specifies whether logging will include the statement text and parameters (if enabled). LogStatement *bool `groups:"create,update" json:"log_statement,omitempty"` - // Specifies whether logging will include the statement text and parameters with the first log entry for a statement/substatement combination or with every entry. + // +kubebuilder:deprecatedversion:warning="log_statement_once is deprecated" + // Deprecated. Specifies whether logging will include the statement text and parameters with the first log entry for a statement/substatement combination or with every entry. LogStatementOnce *bool `groups:"create,update" json:"log_statement_once,omitempty"` // +kubebuilder:validation:MaxLength=64 // +kubebuilder:validation:Pattern=`^[_A-Za-z0-9][-._A-Za-z0-9]{0,63}$` - // Specifies the master role to use for object audit logging. + // +kubebuilder:deprecatedversion:warning="role is deprecated" + // Deprecated. Specifies the master role to use for object audit logging. Role *string `groups:"create,update" json:"role,omitempty"` } @@ -514,7 +527,8 @@ type PgUserConfig struct { // PostgreSQL major version PgVersion *string `groups:"create,update" json:"pg_version,omitempty"` - // System-wide settings for the pgaudit extension + // +kubebuilder:deprecatedversion:warning="pgaudit is deprecated" + // Deprecated. System-wide settings for the pgaudit extension Pgaudit *Pgaudit `groups:"create,update" json:"pgaudit,omitempty"` // PGBouncer connection pooling settings diff --git a/api/v1alpha1/userconfig/service/redis/redis.go b/api/v1alpha1/userconfig/service/redis/redis.go index 2edffd4d..eef58878 100644 --- a/api/v1alpha1/userconfig/service/redis/redis.go +++ b/api/v1alpha1/userconfig/service/redis/redis.go @@ -141,7 +141,7 @@ type RedisUserConfig struct { RedisNumberOfDatabases *int `groups:"create,update" json:"redis_number_of_databases,omitempty"` // +kubebuilder:validation:Enum="off";"rdb" - // When persistence is 'rdb', Redis does RDB dumps each 10 minutes if any key is changed. Also RDB dumps are done according to backup schedule for backup purposes. When persistence is 'off', no RDB dumps and backups are done, so data can be lost at any moment if service is restarted for any reason, or if service is powered off. Also service can't be forked. + // When persistence is 'rdb', Redis does RDB dumps each 10 minutes if any key is changed. Also RDB dumps are done according to the backup schedule for backup purposes. When persistence is 'off', no RDB dumps or backups are done, so data can be lost at any moment if the service is restarted for any reason, or if the service is powered off. Also, the service can't be forked. RedisPersistence *string `groups:"create,update" json:"redis_persistence,omitempty"` // +kubebuilder:validation:Minimum=32 diff --git a/charts/aiven-operator-crds/templates/aiven.io_kafkaconnects.yaml b/charts/aiven-operator-crds/templates/aiven.io_kafkaconnects.yaml index 9e3171d7..89f191c6 100644 --- a/charts/aiven-operator-crds/templates/aiven.io_kafkaconnects.yaml +++ b/charts/aiven-operator-crds/templates/aiven.io_kafkaconnects.yaml @@ -395,6 +395,83 @@ spec: VPC or another type of private network type: boolean type: object + secret_providers: + description: + "Configure external secret providers in order to + reference external secrets in connector configuration. Currently + Hashicorp Vault (provider: vault, auth_method: token) and AWS + Secrets Manager (provider: aws, auth_method: credentials) are + supported. Secrets can be referenced in connector config with + ${::}" + items: + description: SecretProvider + properties: + aws: + description: AWS config for Secret Provider + properties: + access_key: + description: Access key used to authenticate with aws + maxLength: 128 + type: string + auth_method: + description: Auth method of the vault secret provider + enum: + - credentials + type: string + region: + description: + Region used to lookup secrets with AWS + SecretManager + maxLength: 64 + type: string + secret_key: + description: Secret key used to authenticate with aws + maxLength: 128 + type: string + required: + - auth_method + - region + type: object + name: + description: + Name of the secret provider. Used to reference + secrets in connector config. + type: string + vault: + description: Vault Config for Secret Provider + properties: + address: + description: Address of the Vault server + maxLength: 65536 + minLength: 1 + type: string + auth_method: + description: Auth method of the vault secret provider + enum: + - token + type: string + engine_version: + description: + KV Secrets Engine version of the Vault + server instance + enum: + - 1 + - 2 + type: integer + token: + description: + Token used to authenticate with vault and + auth method `token`. + maxLength: 256 + type: string + required: + - address + - auth_method + type: object + required: + - name + type: object + type: array service_log: description: Store logs for the service so that they are available diff --git a/charts/aiven-operator-crds/templates/aiven.io_kafkas.yaml b/charts/aiven-operator-crds/templates/aiven.io_kafkas.yaml index 39aa6e18..48ecc53b 100644 --- a/charts/aiven-operator-crds/templates/aiven.io_kafkas.yaml +++ b/charts/aiven-operator-crds/templates/aiven.io_kafkas.yaml @@ -771,6 +771,83 @@ spec: minimum: 1 type: integer type: object + kafka_connect_secret_providers: + description: + "Configure external secret providers in order to + reference external secrets in connector configuration. Currently + Hashicorp Vault (provider: vault, auth_method: token) and AWS + Secrets Manager (provider: aws, auth_method: credentials) are + supported. Secrets can be referenced in connector config with + ${::}" + items: + description: SecretProvider + properties: + aws: + description: AWS config for Secret Provider + properties: + access_key: + description: Access key used to authenticate with aws + maxLength: 128 + type: string + auth_method: + description: Auth method of the vault secret provider + enum: + - credentials + type: string + region: + description: + Region used to lookup secrets with AWS + SecretManager + maxLength: 64 + type: string + secret_key: + description: Secret key used to authenticate with aws + maxLength: 128 + type: string + required: + - auth_method + - region + type: object + name: + description: + Name of the secret provider. Used to reference + secrets in connector config. + type: string + vault: + description: Vault Config for Secret Provider + properties: + address: + description: Address of the Vault server + maxLength: 65536 + minLength: 1 + type: string + auth_method: + description: Auth method of the vault secret provider + enum: + - token + type: string + engine_version: + description: + KV Secrets Engine version of the Vault + server instance + enum: + - 1 + - 2 + type: integer + token: + description: + Token used to authenticate with vault and + auth method `token`. + maxLength: 256 + type: string + required: + - address + - auth_method + type: object + required: + - name + type: object + type: array kafka_rest: description: Enable Kafka-REST service type: boolean @@ -875,6 +952,9 @@ spec: - "3.6" - "3.7" type: string + letsencrypt_sasl_privatelink: + description: Use Letsencrypt CA for Kafka SASL via Privatelink + type: boolean private_access: description: Allow access to selected service ports from private diff --git a/charts/aiven-operator-crds/templates/aiven.io_postgresqls.yaml b/charts/aiven-operator-crds/templates/aiven.io_postgresqls.yaml index 4d9c811f..2c57a5f0 100644 --- a/charts/aiven-operator-crds/templates/aiven.io_postgresqls.yaml +++ b/charts/aiven-operator-crds/templates/aiven.io_postgresqls.yaml @@ -517,10 +517,7 @@ spec: - VERBOSE type: string log_line_prefix: - description: - Choose from one of the available log-formats. - These can support popular log analyzers like pgbadger, pganalyze - etc. + description: Choose from one of the available log formats. enum: - "'pid=%p,user=%u,db=%d,app=%a,client=%h '" - "'%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h '" @@ -778,99 +775,93 @@ spec: - "16" type: string pgaudit: - description: System-wide settings for the pgaudit extension + description: + Deprecated. System-wide settings for the pgaudit + extension properties: feature_enabled: description: - Enable pgaudit extension. When enabled, pgaudit - extension will be automatically installed.Otherwise, extension - will be uninstalled but auditing configurations will be - preserved. + Deprecated. Enable pgaudit extension. When enabled, + pgaudit extension will be automatically installed.Otherwise, + extension will be uninstalled but auditing configurations + will be preserved. type: boolean log: description: - Specifies which classes of statements will be - logged by session audit logging. + Deprecated. Specifies which classes of statements + will be logged by session audit logging. items: type: string type: array log_catalog: description: - Specifies that session logging should be enabled - in the casewhere all relations in a statement are in pg_catalog. + Deprecated. Specifies that session logging should + be enabled in the casewhere all relations in a statement + are in pg_catalog. type: boolean log_client: description: - Specifies whether log messages will be visible - to a client process such as psql. + Deprecated. Specifies whether log messages will + be visible to a client process such as psql. type: boolean log_level: description: - Specifies the log level that will be used for - log entries. - enum: - - debug1 - - debug2 - - debug3 - - debug4 - - debug5 - - info - - notice - - warning - - log + Deprecated. Specifies the log level that will + be used for log entries. type: string log_max_string_length: description: - Crop parameters representation and whole statements - if they exceed this threshold. A (default) value of -1 disable - the truncation. + Deprecated. Crop parameters representation and + whole statements if they exceed this threshold. A (default) + value of -1 disable the truncation. maximum: 102400 minimum: -1 type: integer log_nested_statements: description: - This GUC allows to turn off logging nested statements, - that is, statements that are executed as part of another - ExecutorRun. + Deprecated. This GUC allows to turn off logging + nested statements, that is, statements that are executed + as part of another ExecutorRun. type: boolean log_parameter: description: - Specifies that audit logging should include the - parameters that were passed with the statement. + Deprecated. Specifies that audit logging should + include the parameters that were passed with the statement. type: boolean log_parameter_max_size: description: - Specifies that parameter values longer than this - setting (in bytes) should not be logged, but replaced with - . + Deprecated. Specifies that parameter values longer + than this setting (in bytes) should not be logged, but replaced + with . type: integer log_relation: description: - Specifies whether session audit logging should - create a separate log entry for each relation (TABLE, VIEW, - etc.) referenced in a SELECT or DML statement. + Deprecated. Specifies whether session audit logging + should create a separate log entry for each relation (TABLE, + VIEW, etc.) referenced in a SELECT or DML statement. type: boolean log_rows: description: - Specifies that audit logging should include the - rows retrieved or affected by a statement. When enabled - the rows field will be included after the parameter field. + Deprecated. Specifies that audit logging should + include the rows retrieved or affected by a statement. When + enabled the rows field will be included after the parameter + field. type: boolean log_statement: description: - Specifies whether logging will include the statement - text and parameters (if enabled). + Deprecated. Specifies whether logging will include + the statement text and parameters (if enabled). type: boolean log_statement_once: description: - Specifies whether logging will include the statement - text and parameters with the first log entry for a statement/substatement - combination or with every entry. + Deprecated. Specifies whether logging will include + the statement text and parameters with the first log entry + for a statement/substatement combination or with every entry. type: boolean role: description: - Specifies the master role to use for object audit - logging. + Deprecated. Specifies the master role to use + for object audit logging. maxLength: 64 pattern: ^[_A-Za-z0-9][-._A-Za-z0-9]{0,63}$ type: string diff --git a/charts/aiven-operator-crds/templates/aiven.io_redis.yaml b/charts/aiven-operator-crds/templates/aiven.io_redis.yaml index 6baa3ae3..f62ba01a 100644 --- a/charts/aiven-operator-crds/templates/aiven.io_redis.yaml +++ b/charts/aiven-operator-crds/templates/aiven.io_redis.yaml @@ -415,10 +415,10 @@ spec: description: When persistence is 'rdb', Redis does RDB dumps each 10 minutes if any key is changed. Also RDB dumps are done according - to backup schedule for backup purposes. When persistence is - 'off', no RDB dumps and backups are done, so data can be lost - at any moment if service is restarted for any reason, or if - service is powered off. Also service can't be forked. + to the backup schedule for backup purposes. When persistence + is 'off', no RDB dumps or backups are done, so data can be lost + at any moment if the service is restarted for any reason, or + if the service is powered off. Also, the service can't be forked. enum: - "off" - rdb diff --git a/charts/aiven-operator-crds/templates/aiven.io_serviceintegrationendpoints.yaml b/charts/aiven-operator-crds/templates/aiven.io_serviceintegrationendpoints.yaml index 565018bd..2893db2a 100644 --- a/charts/aiven-operator-crds/templates/aiven.io_serviceintegrationendpoints.yaml +++ b/charts/aiven-operator-crds/templates/aiven.io_serviceintegrationendpoints.yaml @@ -416,11 +416,11 @@ spec: maxLength: 16384 type: string ssl_mode: - description: SSL Mode + description: + SSL mode to use for the connection. Please note + that Aiven requires TLS for all connections to external PostgreSQL + services. enum: - - disable - - allow - - prefer - require - verify-ca - verify-full diff --git a/charts/aiven-operator-crds/templates/aiven.io_serviceintegrations.yaml b/charts/aiven-operator-crds/templates/aiven.io_serviceintegrations.yaml index 4a16b6a1..3ebab27b 100644 --- a/charts/aiven-operator-crds/templates/aiven.io_serviceintegrations.yaml +++ b/charts/aiven-operator-crds/templates/aiven.io_serviceintegrations.yaml @@ -130,6 +130,7 @@ spec: - TabSeparated - RawBLOB - AvroConfluent + - Parquet type: string date_time_input_format: description: Method to read DateTime from text input formats @@ -299,6 +300,12 @@ spec: maximum: 100000 minimum: 10 type: integer + mirrormaker_custom_metrics: + description: List of custom metrics + items: + type: string + maxItems: 1024 + type: array opensearch: description: Datadog Opensearch Options properties: @@ -506,6 +513,16 @@ spec: kafka_mirrormaker: description: Kafka MirrorMaker configuration values properties: + consumer_auto_offset_reset: + description: + "Set where consumer starts to consume data. Value + `earliest`: Start replication from the earliest offset. + Value `latest`: Start replication from the latest offset. + Default is `earliest`." + enum: + - earliest + - latest + type: string consumer_fetch_min_bytes: description: The minimum amount of data the server should @@ -513,6 +530,13 @@ spec: maximum: 5242880 minimum: 1 type: integer + consumer_max_poll_records: + description: + Set consumer max.poll.records. The default is + 500. + maximum: 20000 + minimum: 100 + type: integer producer_batch_size: description: The batch size in bytes producer will attempt diff --git a/config/crd/bases/aiven.io_kafkaconnects.yaml b/config/crd/bases/aiven.io_kafkaconnects.yaml index 9e3171d7..89f191c6 100644 --- a/config/crd/bases/aiven.io_kafkaconnects.yaml +++ b/config/crd/bases/aiven.io_kafkaconnects.yaml @@ -395,6 +395,83 @@ spec: VPC or another type of private network type: boolean type: object + secret_providers: + description: + "Configure external secret providers in order to + reference external secrets in connector configuration. Currently + Hashicorp Vault (provider: vault, auth_method: token) and AWS + Secrets Manager (provider: aws, auth_method: credentials) are + supported. Secrets can be referenced in connector config with + ${::}" + items: + description: SecretProvider + properties: + aws: + description: AWS config for Secret Provider + properties: + access_key: + description: Access key used to authenticate with aws + maxLength: 128 + type: string + auth_method: + description: Auth method of the vault secret provider + enum: + - credentials + type: string + region: + description: + Region used to lookup secrets with AWS + SecretManager + maxLength: 64 + type: string + secret_key: + description: Secret key used to authenticate with aws + maxLength: 128 + type: string + required: + - auth_method + - region + type: object + name: + description: + Name of the secret provider. Used to reference + secrets in connector config. + type: string + vault: + description: Vault Config for Secret Provider + properties: + address: + description: Address of the Vault server + maxLength: 65536 + minLength: 1 + type: string + auth_method: + description: Auth method of the vault secret provider + enum: + - token + type: string + engine_version: + description: + KV Secrets Engine version of the Vault + server instance + enum: + - 1 + - 2 + type: integer + token: + description: + Token used to authenticate with vault and + auth method `token`. + maxLength: 256 + type: string + required: + - address + - auth_method + type: object + required: + - name + type: object + type: array service_log: description: Store logs for the service so that they are available diff --git a/config/crd/bases/aiven.io_kafkas.yaml b/config/crd/bases/aiven.io_kafkas.yaml index 39aa6e18..48ecc53b 100644 --- a/config/crd/bases/aiven.io_kafkas.yaml +++ b/config/crd/bases/aiven.io_kafkas.yaml @@ -771,6 +771,83 @@ spec: minimum: 1 type: integer type: object + kafka_connect_secret_providers: + description: + "Configure external secret providers in order to + reference external secrets in connector configuration. Currently + Hashicorp Vault (provider: vault, auth_method: token) and AWS + Secrets Manager (provider: aws, auth_method: credentials) are + supported. Secrets can be referenced in connector config with + ${::}" + items: + description: SecretProvider + properties: + aws: + description: AWS config for Secret Provider + properties: + access_key: + description: Access key used to authenticate with aws + maxLength: 128 + type: string + auth_method: + description: Auth method of the vault secret provider + enum: + - credentials + type: string + region: + description: + Region used to lookup secrets with AWS + SecretManager + maxLength: 64 + type: string + secret_key: + description: Secret key used to authenticate with aws + maxLength: 128 + type: string + required: + - auth_method + - region + type: object + name: + description: + Name of the secret provider. Used to reference + secrets in connector config. + type: string + vault: + description: Vault Config for Secret Provider + properties: + address: + description: Address of the Vault server + maxLength: 65536 + minLength: 1 + type: string + auth_method: + description: Auth method of the vault secret provider + enum: + - token + type: string + engine_version: + description: + KV Secrets Engine version of the Vault + server instance + enum: + - 1 + - 2 + type: integer + token: + description: + Token used to authenticate with vault and + auth method `token`. + maxLength: 256 + type: string + required: + - address + - auth_method + type: object + required: + - name + type: object + type: array kafka_rest: description: Enable Kafka-REST service type: boolean @@ -875,6 +952,9 @@ spec: - "3.6" - "3.7" type: string + letsencrypt_sasl_privatelink: + description: Use Letsencrypt CA for Kafka SASL via Privatelink + type: boolean private_access: description: Allow access to selected service ports from private diff --git a/config/crd/bases/aiven.io_postgresqls.yaml b/config/crd/bases/aiven.io_postgresqls.yaml index 4d9c811f..2c57a5f0 100644 --- a/config/crd/bases/aiven.io_postgresqls.yaml +++ b/config/crd/bases/aiven.io_postgresqls.yaml @@ -517,10 +517,7 @@ spec: - VERBOSE type: string log_line_prefix: - description: - Choose from one of the available log-formats. - These can support popular log analyzers like pgbadger, pganalyze - etc. + description: Choose from one of the available log formats. enum: - "'pid=%p,user=%u,db=%d,app=%a,client=%h '" - "'%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h '" @@ -778,99 +775,93 @@ spec: - "16" type: string pgaudit: - description: System-wide settings for the pgaudit extension + description: + Deprecated. System-wide settings for the pgaudit + extension properties: feature_enabled: description: - Enable pgaudit extension. When enabled, pgaudit - extension will be automatically installed.Otherwise, extension - will be uninstalled but auditing configurations will be - preserved. + Deprecated. Enable pgaudit extension. When enabled, + pgaudit extension will be automatically installed.Otherwise, + extension will be uninstalled but auditing configurations + will be preserved. type: boolean log: description: - Specifies which classes of statements will be - logged by session audit logging. + Deprecated. Specifies which classes of statements + will be logged by session audit logging. items: type: string type: array log_catalog: description: - Specifies that session logging should be enabled - in the casewhere all relations in a statement are in pg_catalog. + Deprecated. Specifies that session logging should + be enabled in the casewhere all relations in a statement + are in pg_catalog. type: boolean log_client: description: - Specifies whether log messages will be visible - to a client process such as psql. + Deprecated. Specifies whether log messages will + be visible to a client process such as psql. type: boolean log_level: description: - Specifies the log level that will be used for - log entries. - enum: - - debug1 - - debug2 - - debug3 - - debug4 - - debug5 - - info - - notice - - warning - - log + Deprecated. Specifies the log level that will + be used for log entries. type: string log_max_string_length: description: - Crop parameters representation and whole statements - if they exceed this threshold. A (default) value of -1 disable - the truncation. + Deprecated. Crop parameters representation and + whole statements if they exceed this threshold. A (default) + value of -1 disable the truncation. maximum: 102400 minimum: -1 type: integer log_nested_statements: description: - This GUC allows to turn off logging nested statements, - that is, statements that are executed as part of another - ExecutorRun. + Deprecated. This GUC allows to turn off logging + nested statements, that is, statements that are executed + as part of another ExecutorRun. type: boolean log_parameter: description: - Specifies that audit logging should include the - parameters that were passed with the statement. + Deprecated. Specifies that audit logging should + include the parameters that were passed with the statement. type: boolean log_parameter_max_size: description: - Specifies that parameter values longer than this - setting (in bytes) should not be logged, but replaced with - . + Deprecated. Specifies that parameter values longer + than this setting (in bytes) should not be logged, but replaced + with . type: integer log_relation: description: - Specifies whether session audit logging should - create a separate log entry for each relation (TABLE, VIEW, - etc.) referenced in a SELECT or DML statement. + Deprecated. Specifies whether session audit logging + should create a separate log entry for each relation (TABLE, + VIEW, etc.) referenced in a SELECT or DML statement. type: boolean log_rows: description: - Specifies that audit logging should include the - rows retrieved or affected by a statement. When enabled - the rows field will be included after the parameter field. + Deprecated. Specifies that audit logging should + include the rows retrieved or affected by a statement. When + enabled the rows field will be included after the parameter + field. type: boolean log_statement: description: - Specifies whether logging will include the statement - text and parameters (if enabled). + Deprecated. Specifies whether logging will include + the statement text and parameters (if enabled). type: boolean log_statement_once: description: - Specifies whether logging will include the statement - text and parameters with the first log entry for a statement/substatement - combination or with every entry. + Deprecated. Specifies whether logging will include + the statement text and parameters with the first log entry + for a statement/substatement combination or with every entry. type: boolean role: description: - Specifies the master role to use for object audit - logging. + Deprecated. Specifies the master role to use + for object audit logging. maxLength: 64 pattern: ^[_A-Za-z0-9][-._A-Za-z0-9]{0,63}$ type: string diff --git a/config/crd/bases/aiven.io_redis.yaml b/config/crd/bases/aiven.io_redis.yaml index 6baa3ae3..f62ba01a 100644 --- a/config/crd/bases/aiven.io_redis.yaml +++ b/config/crd/bases/aiven.io_redis.yaml @@ -415,10 +415,10 @@ spec: description: When persistence is 'rdb', Redis does RDB dumps each 10 minutes if any key is changed. Also RDB dumps are done according - to backup schedule for backup purposes. When persistence is - 'off', no RDB dumps and backups are done, so data can be lost - at any moment if service is restarted for any reason, or if - service is powered off. Also service can't be forked. + to the backup schedule for backup purposes. When persistence + is 'off', no RDB dumps or backups are done, so data can be lost + at any moment if the service is restarted for any reason, or + if the service is powered off. Also, the service can't be forked. enum: - "off" - rdb diff --git a/config/crd/bases/aiven.io_serviceintegrationendpoints.yaml b/config/crd/bases/aiven.io_serviceintegrationendpoints.yaml index 565018bd..2893db2a 100644 --- a/config/crd/bases/aiven.io_serviceintegrationendpoints.yaml +++ b/config/crd/bases/aiven.io_serviceintegrationendpoints.yaml @@ -416,11 +416,11 @@ spec: maxLength: 16384 type: string ssl_mode: - description: SSL Mode + description: + SSL mode to use for the connection. Please note + that Aiven requires TLS for all connections to external PostgreSQL + services. enum: - - disable - - allow - - prefer - require - verify-ca - verify-full diff --git a/config/crd/bases/aiven.io_serviceintegrations.yaml b/config/crd/bases/aiven.io_serviceintegrations.yaml index 4a16b6a1..3ebab27b 100644 --- a/config/crd/bases/aiven.io_serviceintegrations.yaml +++ b/config/crd/bases/aiven.io_serviceintegrations.yaml @@ -130,6 +130,7 @@ spec: - TabSeparated - RawBLOB - AvroConfluent + - Parquet type: string date_time_input_format: description: Method to read DateTime from text input formats @@ -299,6 +300,12 @@ spec: maximum: 100000 minimum: 10 type: integer + mirrormaker_custom_metrics: + description: List of custom metrics + items: + type: string + maxItems: 1024 + type: array opensearch: description: Datadog Opensearch Options properties: @@ -506,6 +513,16 @@ spec: kafka_mirrormaker: description: Kafka MirrorMaker configuration values properties: + consumer_auto_offset_reset: + description: + "Set where consumer starts to consume data. Value + `earliest`: Start replication from the earliest offset. + Value `latest`: Start replication from the latest offset. + Default is `earliest`." + enum: + - earliest + - latest + type: string consumer_fetch_min_bytes: description: The minimum amount of data the server should @@ -513,6 +530,13 @@ spec: maximum: 5242880 minimum: 1 type: integer + consumer_max_poll_records: + description: + Set consumer max.poll.records. The default is + 500. + maximum: 20000 + minimum: 100 + type: integer producer_batch_size: description: The batch size in bytes producer will attempt diff --git a/docs/docs/api-reference/kafka.md b/docs/docs/api-reference/kafka.md index 081f53e5..99a4cc0e 100644 --- a/docs/docs/api-reference/kafka.md +++ b/docs/docs/api-reference/kafka.md @@ -157,10 +157,12 @@ Kafka specific user configuration options. - [`kafka_authentication_methods`](#spec.userConfig.kafka_authentication_methods-property){: name='spec.userConfig.kafka_authentication_methods-property'} (object). Kafka authentication methods. See below for [nested schema](#spec.userConfig.kafka_authentication_methods). - [`kafka_connect`](#spec.userConfig.kafka_connect-property){: name='spec.userConfig.kafka_connect-property'} (boolean). Enable Kafka Connect service. - [`kafka_connect_config`](#spec.userConfig.kafka_connect_config-property){: name='spec.userConfig.kafka_connect_config-property'} (object). Kafka Connect configuration values. See below for [nested schema](#spec.userConfig.kafka_connect_config). +- [`kafka_connect_secret_providers`](#spec.userConfig.kafka_connect_secret_providers-property){: name='spec.userConfig.kafka_connect_secret_providers-property'} (array of objects). Configure external secret providers in order to reference external secrets in connector configuration. Currently Hashicorp Vault (provider: vault, auth_method: token) and AWS Secrets Manager (provider: aws, auth_method: credentials) are supported. Secrets can be referenced in connector config with ${::}. See below for [nested schema](#spec.userConfig.kafka_connect_secret_providers). - [`kafka_rest`](#spec.userConfig.kafka_rest-property){: name='spec.userConfig.kafka_rest-property'} (boolean). Enable Kafka-REST service. - [`kafka_rest_authorization`](#spec.userConfig.kafka_rest_authorization-property){: name='spec.userConfig.kafka_rest_authorization-property'} (boolean). Enable authorization in Kafka-REST service. - [`kafka_rest_config`](#spec.userConfig.kafka_rest_config-property){: name='spec.userConfig.kafka_rest_config-property'} (object). Kafka REST configuration. See below for [nested schema](#spec.userConfig.kafka_rest_config). - [`kafka_version`](#spec.userConfig.kafka_version-property){: name='spec.userConfig.kafka_version-property'} (string, Enum: `3.4`, `3.5`, `3.6`, `3.7`). Kafka major version. +- [`letsencrypt_sasl_privatelink`](#spec.userConfig.letsencrypt_sasl_privatelink-property){: name='spec.userConfig.letsencrypt_sasl_privatelink-property'} (boolean). Use Letsencrypt CA for Kafka SASL via Privatelink. - [`private_access`](#spec.userConfig.private_access-property){: name='spec.userConfig.private_access-property'} (object). Allow access to selected service ports from private networks. See below for [nested schema](#spec.userConfig.private_access). - [`privatelink_access`](#spec.userConfig.privatelink_access-property){: name='spec.userConfig.privatelink_access-property'} (object). Allow access to selected service components through Privatelink. See below for [nested schema](#spec.userConfig.privatelink_access). - [`public_access`](#spec.userConfig.public_access-property){: name='spec.userConfig.public_access-property'} (object). Allow access to selected service ports from the public Internet. See below for [nested schema](#spec.userConfig.public_access). @@ -275,6 +277,53 @@ Kafka Connect configuration values. - [`scheduled_rebalance_max_delay_ms`](#spec.userConfig.kafka_connect_config.scheduled_rebalance_max_delay_ms-property){: name='spec.userConfig.kafka_connect_config.scheduled_rebalance_max_delay_ms-property'} (integer, Minimum: 0, Maximum: 600000). The maximum delay that is scheduled in order to wait for the return of one or more departed workers before rebalancing and reassigning their connectors and tasks to the group. During this period the connectors and tasks of the departed workers remain unassigned. Defaults to 5 minutes. - [`session_timeout_ms`](#spec.userConfig.kafka_connect_config.session_timeout_ms-property){: name='spec.userConfig.kafka_connect_config.session_timeout_ms-property'} (integer, Minimum: 1, Maximum: 2147483647). The timeout in milliseconds used to detect failures when using Kafka’s group management facilities (defaults to 10000). +### kafka_connect_secret_providers {: #spec.userConfig.kafka_connect_secret_providers } + +_Appears on [`spec.userConfig`](#spec.userConfig)._ + +Configure external secret providers in order to reference external secrets in connector configuration. Currently Hashicorp Vault (provider: vault, auth_method: token) and AWS Secrets Manager (provider: aws, auth_method: credentials) are supported. Secrets can be referenced in connector config with ${::}. + +**Required** + +- [`name`](#spec.userConfig.kafka_connect_secret_providers.name-property){: name='spec.userConfig.kafka_connect_secret_providers.name-property'} (string). Name of the secret provider. Used to reference secrets in connector config. + +**Optional** + +- [`aws`](#spec.userConfig.kafka_connect_secret_providers.aws-property){: name='spec.userConfig.kafka_connect_secret_providers.aws-property'} (object). AWS config for Secret Provider. See below for [nested schema](#spec.userConfig.kafka_connect_secret_providers.aws). +- [`vault`](#spec.userConfig.kafka_connect_secret_providers.vault-property){: name='spec.userConfig.kafka_connect_secret_providers.vault-property'} (object). Vault Config for Secret Provider. See below for [nested schema](#spec.userConfig.kafka_connect_secret_providers.vault). + +#### aws {: #spec.userConfig.kafka_connect_secret_providers.aws } + +_Appears on [`spec.userConfig.kafka_connect_secret_providers`](#spec.userConfig.kafka_connect_secret_providers)._ + +AWS config for Secret Provider. + +**Required** + +- [`auth_method`](#spec.userConfig.kafka_connect_secret_providers.aws.auth_method-property){: name='spec.userConfig.kafka_connect_secret_providers.aws.auth_method-property'} (string, Enum: `credentials`). Auth method of the vault secret provider. +- [`region`](#spec.userConfig.kafka_connect_secret_providers.aws.region-property){: name='spec.userConfig.kafka_connect_secret_providers.aws.region-property'} (string, MaxLength: 64). Region used to lookup secrets with AWS SecretManager. + +**Optional** + +- [`access_key`](#spec.userConfig.kafka_connect_secret_providers.aws.access_key-property){: name='spec.userConfig.kafka_connect_secret_providers.aws.access_key-property'} (string, MaxLength: 128). Access key used to authenticate with aws. +- [`secret_key`](#spec.userConfig.kafka_connect_secret_providers.aws.secret_key-property){: name='spec.userConfig.kafka_connect_secret_providers.aws.secret_key-property'} (string, MaxLength: 128). Secret key used to authenticate with aws. + +#### vault {: #spec.userConfig.kafka_connect_secret_providers.vault } + +_Appears on [`spec.userConfig.kafka_connect_secret_providers`](#spec.userConfig.kafka_connect_secret_providers)._ + +Vault Config for Secret Provider. + +**Required** + +- [`address`](#spec.userConfig.kafka_connect_secret_providers.vault.address-property){: name='spec.userConfig.kafka_connect_secret_providers.vault.address-property'} (string, MinLength: 1, MaxLength: 65536). Address of the Vault server. +- [`auth_method`](#spec.userConfig.kafka_connect_secret_providers.vault.auth_method-property){: name='spec.userConfig.kafka_connect_secret_providers.vault.auth_method-property'} (string, Enum: `token`). Auth method of the vault secret provider. + +**Optional** + +- [`engine_version`](#spec.userConfig.kafka_connect_secret_providers.vault.engine_version-property){: name='spec.userConfig.kafka_connect_secret_providers.vault.engine_version-property'} (integer, Enum: `1`, `2`). KV Secrets Engine version of the Vault server instance. +- [`token`](#spec.userConfig.kafka_connect_secret_providers.vault.token-property){: name='spec.userConfig.kafka_connect_secret_providers.vault.token-property'} (string, MaxLength: 256). Token used to authenticate with vault and auth method `token`. + ### kafka_rest_config {: #spec.userConfig.kafka_rest_config } _Appears on [`spec.userConfig`](#spec.userConfig)._ diff --git a/docs/docs/api-reference/kafkaconnect.md b/docs/docs/api-reference/kafkaconnect.md index d7a77453..2de83cc1 100644 --- a/docs/docs/api-reference/kafkaconnect.md +++ b/docs/docs/api-reference/kafkaconnect.md @@ -122,6 +122,7 @@ KafkaConnect specific user configuration options. - [`private_access`](#spec.userConfig.private_access-property){: name='spec.userConfig.private_access-property'} (object). Allow access to selected service ports from private networks. See below for [nested schema](#spec.userConfig.private_access). - [`privatelink_access`](#spec.userConfig.privatelink_access-property){: name='spec.userConfig.privatelink_access-property'} (object). Allow access to selected service components through Privatelink. See below for [nested schema](#spec.userConfig.privatelink_access). - [`public_access`](#spec.userConfig.public_access-property){: name='spec.userConfig.public_access-property'} (object). Allow access to selected service ports from the public Internet. See below for [nested schema](#spec.userConfig.public_access). +- [`secret_providers`](#spec.userConfig.secret_providers-property){: name='spec.userConfig.secret_providers-property'} (array of objects). Configure external secret providers in order to reference external secrets in connector configuration. Currently Hashicorp Vault (provider: vault, auth_method: token) and AWS Secrets Manager (provider: aws, auth_method: credentials) are supported. Secrets can be referenced in connector config with ${::}. See below for [nested schema](#spec.userConfig.secret_providers). - [`service_log`](#spec.userConfig.service_log-property){: name='spec.userConfig.service_log-property'} (boolean). Store logs for the service so that they are available in the HTTP API and console. - [`static_ips`](#spec.userConfig.static_ips-property){: name='spec.userConfig.static_ips-property'} (boolean). Use static public IP addresses. @@ -197,3 +198,51 @@ Allow access to selected service ports from the public Internet. - [`kafka_connect`](#spec.userConfig.public_access.kafka_connect-property){: name='spec.userConfig.public_access.kafka_connect-property'} (boolean). Allow clients to connect to kafka_connect from the public internet for service nodes that are in a project VPC or another type of private network. - [`prometheus`](#spec.userConfig.public_access.prometheus-property){: name='spec.userConfig.public_access.prometheus-property'} (boolean). Allow clients to connect to prometheus from the public internet for service nodes that are in a project VPC or another type of private network. + +### secret_providers {: #spec.userConfig.secret_providers } + +_Appears on [`spec.userConfig`](#spec.userConfig)._ + +Configure external secret providers in order to reference external secrets in connector configuration. Currently Hashicorp Vault (provider: vault, auth_method: token) and AWS Secrets Manager (provider: aws, auth_method: credentials) are supported. Secrets can be referenced in connector config with ${::}. + +**Required** + +- [`name`](#spec.userConfig.secret_providers.name-property){: name='spec.userConfig.secret_providers.name-property'} (string). Name of the secret provider. Used to reference secrets in connector config. + +**Optional** + +- [`aws`](#spec.userConfig.secret_providers.aws-property){: name='spec.userConfig.secret_providers.aws-property'} (object). AWS config for Secret Provider. See below for [nested schema](#spec.userConfig.secret_providers.aws). +- [`vault`](#spec.userConfig.secret_providers.vault-property){: name='spec.userConfig.secret_providers.vault-property'} (object). Vault Config for Secret Provider. See below for [nested schema](#spec.userConfig.secret_providers.vault). + +#### aws {: #spec.userConfig.secret_providers.aws } + +_Appears on [`spec.userConfig.secret_providers`](#spec.userConfig.secret_providers)._ + +AWS config for Secret Provider. + +**Required** + +- [`auth_method`](#spec.userConfig.secret_providers.aws.auth_method-property){: name='spec.userConfig.secret_providers.aws.auth_method-property'} (string, Enum: `credentials`). Auth method of the vault secret provider. +- [`region`](#spec.userConfig.secret_providers.aws.region-property){: name='spec.userConfig.secret_providers.aws.region-property'} (string, MaxLength: 64). Region used to lookup secrets with AWS SecretManager. + +**Optional** + +- [`access_key`](#spec.userConfig.secret_providers.aws.access_key-property){: name='spec.userConfig.secret_providers.aws.access_key-property'} (string, MaxLength: 128). Access key used to authenticate with aws. +- [`secret_key`](#spec.userConfig.secret_providers.aws.secret_key-property){: name='spec.userConfig.secret_providers.aws.secret_key-property'} (string, MaxLength: 128). Secret key used to authenticate with aws. + +#### vault {: #spec.userConfig.secret_providers.vault } + +_Appears on [`spec.userConfig.secret_providers`](#spec.userConfig.secret_providers)._ + +Vault Config for Secret Provider. + +**Required** + +- [`address`](#spec.userConfig.secret_providers.vault.address-property){: name='spec.userConfig.secret_providers.vault.address-property'} (string, MinLength: 1, MaxLength: 65536). Address of the Vault server. +- [`auth_method`](#spec.userConfig.secret_providers.vault.auth_method-property){: name='spec.userConfig.secret_providers.vault.auth_method-property'} (string, Enum: `token`). Auth method of the vault secret provider. + +**Optional** + +- [`engine_version`](#spec.userConfig.secret_providers.vault.engine_version-property){: name='spec.userConfig.secret_providers.vault.engine_version-property'} (integer, Enum: `1`, `2`). KV Secrets Engine version of the Vault server instance. +- [`token`](#spec.userConfig.secret_providers.vault.token-property){: name='spec.userConfig.secret_providers.vault.token-property'} (string, MaxLength: 256). Token used to authenticate with vault and auth method `token`. + diff --git a/docs/docs/api-reference/postgresql.md b/docs/docs/api-reference/postgresql.md index f3930c66..88c20446 100644 --- a/docs/docs/api-reference/postgresql.md +++ b/docs/docs/api-reference/postgresql.md @@ -165,7 +165,7 @@ PostgreSQL specific user configuration options. - [`pg_service_to_fork_from`](#spec.userConfig.pg_service_to_fork_from-property){: name='spec.userConfig.pg_service_to_fork_from-property'} (string, Immutable, Pattern: `^[a-z][-a-z0-9]{0,63}$|^$`, MaxLength: 64). Name of the PG Service from which to fork (deprecated, use service_to_fork_from). This has effect only when a new service is being created. - [`pg_stat_monitor_enable`](#spec.userConfig.pg_stat_monitor_enable-property){: name='spec.userConfig.pg_stat_monitor_enable-property'} (boolean). Enable the pg_stat_monitor extension. Enabling this extension will cause the cluster to be restarted.When this extension is enabled, pg_stat_statements results for utility commands are unreliable. - [`pg_version`](#spec.userConfig.pg_version-property){: name='spec.userConfig.pg_version-property'} (string, Enum: `12`, `13`, `14`, `15`, `16`). PostgreSQL major version. -- [`pgaudit`](#spec.userConfig.pgaudit-property){: name='spec.userConfig.pgaudit-property'} (object). System-wide settings for the pgaudit extension. See below for [nested schema](#spec.userConfig.pgaudit). +- [`pgaudit`](#spec.userConfig.pgaudit-property){: name='spec.userConfig.pgaudit-property'} (object). Deprecated. System-wide settings for the pgaudit extension. See below for [nested schema](#spec.userConfig.pgaudit). - [`pgbouncer`](#spec.userConfig.pgbouncer-property){: name='spec.userConfig.pgbouncer-property'} (object). PGBouncer connection pooling settings. See below for [nested schema](#spec.userConfig.pgbouncer). - [`pglookout`](#spec.userConfig.pglookout-property){: name='spec.userConfig.pglookout-property'} (object). System-wide settings for pglookout. See below for [nested schema](#spec.userConfig.pglookout). - [`private_access`](#spec.userConfig.private_access-property){: name='spec.userConfig.private_access-property'} (object). Allow access to selected service ports from private networks. See below for [nested schema](#spec.userConfig.private_access). @@ -243,7 +243,7 @@ postgresql.conf configuration values. - [`jit`](#spec.userConfig.pg.jit-property){: name='spec.userConfig.pg.jit-property'} (boolean). Controls system-wide use of Just-in-Time Compilation (JIT). - [`log_autovacuum_min_duration`](#spec.userConfig.pg.log_autovacuum_min_duration-property){: name='spec.userConfig.pg.log_autovacuum_min_duration-property'} (integer, Minimum: -1, Maximum: 2147483647). Causes each action executed by autovacuum to be logged if it ran for at least the specified number of milliseconds. Setting this to zero logs all autovacuum actions. Minus-one (the default) disables logging autovacuum actions. - [`log_error_verbosity`](#spec.userConfig.pg.log_error_verbosity-property){: name='spec.userConfig.pg.log_error_verbosity-property'} (string, Enum: `TERSE`, `DEFAULT`, `VERBOSE`). Controls the amount of detail written in the server log for each message that is logged. -- [`log_line_prefix`](#spec.userConfig.pg.log_line_prefix-property){: name='spec.userConfig.pg.log_line_prefix-property'} (string, Enum: `'pid=%p,user=%u,db=%d,app=%a,client=%h '`, `'%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h '`, `'%m [%p] %q[user=%u,db=%d,app=%a] '`). Choose from one of the available log-formats. These can support popular log analyzers like pgbadger, pganalyze etc. +- [`log_line_prefix`](#spec.userConfig.pg.log_line_prefix-property){: name='spec.userConfig.pg.log_line_prefix-property'} (string, Enum: `'pid=%p,user=%u,db=%d,app=%a,client=%h '`, `'%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h '`, `'%m [%p] %q[user=%u,db=%d,app=%a] '`). Choose from one of the available log formats. - [`log_min_duration_statement`](#spec.userConfig.pg.log_min_duration_statement-property){: name='spec.userConfig.pg.log_min_duration_statement-property'} (integer, Minimum: -1, Maximum: 86400000). Log statements that take more than this number of milliseconds to run, -1 disables. - [`log_temp_files`](#spec.userConfig.pg.log_temp_files-property){: name='spec.userConfig.pg.log_temp_files-property'} (integer, Minimum: -1, Maximum: 2147483647). Log statements for each temporary file created larger than this number of kilobytes, -1 disables. - [`max_files_per_process`](#spec.userConfig.pg.max_files_per_process-property){: name='spec.userConfig.pg.max_files_per_process-property'} (integer, Minimum: 1000, Maximum: 4096). PostgreSQL maximum number of files that can be open per process. @@ -292,24 +292,24 @@ Deprecated. System-wide settings for the pg_qualstats extension. _Appears on [`spec.userConfig`](#spec.userConfig)._ -System-wide settings for the pgaudit extension. +Deprecated. System-wide settings for the pgaudit extension. **Optional** -- [`feature_enabled`](#spec.userConfig.pgaudit.feature_enabled-property){: name='spec.userConfig.pgaudit.feature_enabled-property'} (boolean). Enable pgaudit extension. When enabled, pgaudit extension will be automatically installed.Otherwise, extension will be uninstalled but auditing configurations will be preserved. -- [`log`](#spec.userConfig.pgaudit.log-property){: name='spec.userConfig.pgaudit.log-property'} (array of strings). Specifies which classes of statements will be logged by session audit logging. -- [`log_catalog`](#spec.userConfig.pgaudit.log_catalog-property){: name='spec.userConfig.pgaudit.log_catalog-property'} (boolean). Specifies that session logging should be enabled in the casewhere all relations in a statement are in pg_catalog. -- [`log_client`](#spec.userConfig.pgaudit.log_client-property){: name='spec.userConfig.pgaudit.log_client-property'} (boolean). Specifies whether log messages will be visible to a client process such as psql. -- [`log_level`](#spec.userConfig.pgaudit.log_level-property){: name='spec.userConfig.pgaudit.log_level-property'} (string, Enum: `debug1`, `debug2`, `debug3`, `debug4`, `debug5`, `info`, `notice`, `warning`, `log`). Specifies the log level that will be used for log entries. -- [`log_max_string_length`](#spec.userConfig.pgaudit.log_max_string_length-property){: name='spec.userConfig.pgaudit.log_max_string_length-property'} (integer, Minimum: -1, Maximum: 102400). Crop parameters representation and whole statements if they exceed this threshold. A (default) value of -1 disable the truncation. -- [`log_nested_statements`](#spec.userConfig.pgaudit.log_nested_statements-property){: name='spec.userConfig.pgaudit.log_nested_statements-property'} (boolean). This GUC allows to turn off logging nested statements, that is, statements that are executed as part of another ExecutorRun. -- [`log_parameter`](#spec.userConfig.pgaudit.log_parameter-property){: name='spec.userConfig.pgaudit.log_parameter-property'} (boolean). Specifies that audit logging should include the parameters that were passed with the statement. -- [`log_parameter_max_size`](#spec.userConfig.pgaudit.log_parameter_max_size-property){: name='spec.userConfig.pgaudit.log_parameter_max_size-property'} (integer). Specifies that parameter values longer than this setting (in bytes) should not be logged, but replaced with . -- [`log_relation`](#spec.userConfig.pgaudit.log_relation-property){: name='spec.userConfig.pgaudit.log_relation-property'} (boolean). Specifies whether session audit logging should create a separate log entry for each relation (TABLE, VIEW, etc.) referenced in a SELECT or DML statement. -- [`log_rows`](#spec.userConfig.pgaudit.log_rows-property){: name='spec.userConfig.pgaudit.log_rows-property'} (boolean). Specifies that audit logging should include the rows retrieved or affected by a statement. When enabled the rows field will be included after the parameter field. -- [`log_statement`](#spec.userConfig.pgaudit.log_statement-property){: name='spec.userConfig.pgaudit.log_statement-property'} (boolean). Specifies whether logging will include the statement text and parameters (if enabled). -- [`log_statement_once`](#spec.userConfig.pgaudit.log_statement_once-property){: name='spec.userConfig.pgaudit.log_statement_once-property'} (boolean). Specifies whether logging will include the statement text and parameters with the first log entry for a statement/substatement combination or with every entry. -- [`role`](#spec.userConfig.pgaudit.role-property){: name='spec.userConfig.pgaudit.role-property'} (string, Pattern: `^[_A-Za-z0-9][-._A-Za-z0-9]{0,63}$`, MaxLength: 64). Specifies the master role to use for object audit logging. +- [`feature_enabled`](#spec.userConfig.pgaudit.feature_enabled-property){: name='spec.userConfig.pgaudit.feature_enabled-property'} (boolean). Deprecated. Enable pgaudit extension. When enabled, pgaudit extension will be automatically installed.Otherwise, extension will be uninstalled but auditing configurations will be preserved. +- [`log`](#spec.userConfig.pgaudit.log-property){: name='spec.userConfig.pgaudit.log-property'} (array of strings). Deprecated. Specifies which classes of statements will be logged by session audit logging. +- [`log_catalog`](#spec.userConfig.pgaudit.log_catalog-property){: name='spec.userConfig.pgaudit.log_catalog-property'} (boolean). Deprecated. Specifies that session logging should be enabled in the casewhere all relations in a statement are in pg_catalog. +- [`log_client`](#spec.userConfig.pgaudit.log_client-property){: name='spec.userConfig.pgaudit.log_client-property'} (boolean). Deprecated. Specifies whether log messages will be visible to a client process such as psql. +- [`log_level`](#spec.userConfig.pgaudit.log_level-property){: name='spec.userConfig.pgaudit.log_level-property'} (string). Deprecated. Specifies the log level that will be used for log entries. +- [`log_max_string_length`](#spec.userConfig.pgaudit.log_max_string_length-property){: name='spec.userConfig.pgaudit.log_max_string_length-property'} (integer, Minimum: -1, Maximum: 102400). Deprecated. Crop parameters representation and whole statements if they exceed this threshold. A (default) value of -1 disable the truncation. +- [`log_nested_statements`](#spec.userConfig.pgaudit.log_nested_statements-property){: name='spec.userConfig.pgaudit.log_nested_statements-property'} (boolean). Deprecated. This GUC allows to turn off logging nested statements, that is, statements that are executed as part of another ExecutorRun. +- [`log_parameter`](#spec.userConfig.pgaudit.log_parameter-property){: name='spec.userConfig.pgaudit.log_parameter-property'} (boolean). Deprecated. Specifies that audit logging should include the parameters that were passed with the statement. +- [`log_parameter_max_size`](#spec.userConfig.pgaudit.log_parameter_max_size-property){: name='spec.userConfig.pgaudit.log_parameter_max_size-property'} (integer). Deprecated. Specifies that parameter values longer than this setting (in bytes) should not be logged, but replaced with . +- [`log_relation`](#spec.userConfig.pgaudit.log_relation-property){: name='spec.userConfig.pgaudit.log_relation-property'} (boolean). Deprecated. Specifies whether session audit logging should create a separate log entry for each relation (TABLE, VIEW, etc.) referenced in a SELECT or DML statement. +- [`log_rows`](#spec.userConfig.pgaudit.log_rows-property){: name='spec.userConfig.pgaudit.log_rows-property'} (boolean). Deprecated. Specifies that audit logging should include the rows retrieved or affected by a statement. When enabled the rows field will be included after the parameter field. +- [`log_statement`](#spec.userConfig.pgaudit.log_statement-property){: name='spec.userConfig.pgaudit.log_statement-property'} (boolean). Deprecated. Specifies whether logging will include the statement text and parameters (if enabled). +- [`log_statement_once`](#spec.userConfig.pgaudit.log_statement_once-property){: name='spec.userConfig.pgaudit.log_statement_once-property'} (boolean). Deprecated. Specifies whether logging will include the statement text and parameters with the first log entry for a statement/substatement combination or with every entry. +- [`role`](#spec.userConfig.pgaudit.role-property){: name='spec.userConfig.pgaudit.role-property'} (string, Pattern: `^[_A-Za-z0-9][-._A-Za-z0-9]{0,63}$`, MaxLength: 64). Deprecated. Specifies the master role to use for object audit logging. ### pgbouncer {: #spec.userConfig.pgbouncer } diff --git a/docs/docs/api-reference/redis.md b/docs/docs/api-reference/redis.md index 1b8428ed..7276be2b 100644 --- a/docs/docs/api-reference/redis.md +++ b/docs/docs/api-reference/redis.md @@ -166,7 +166,7 @@ Redis specific user configuration options. - [`redis_maxmemory_policy`](#spec.userConfig.redis_maxmemory_policy-property){: name='spec.userConfig.redis_maxmemory_policy-property'} (string, Enum: `noeviction`, `allkeys-lru`, `volatile-lru`, `allkeys-random`, `volatile-random`, `volatile-ttl`, `volatile-lfu`, `allkeys-lfu`). Redis maxmemory-policy. - [`redis_notify_keyspace_events`](#spec.userConfig.redis_notify_keyspace_events-property){: name='spec.userConfig.redis_notify_keyspace_events-property'} (string, Pattern: `^[KEg\$lshzxentdmA]*$`, MaxLength: 32). Set notify-keyspace-events option. - [`redis_number_of_databases`](#spec.userConfig.redis_number_of_databases-property){: name='spec.userConfig.redis_number_of_databases-property'} (integer, Minimum: 1, Maximum: 128). Set number of Redis databases. Changing this will cause a restart of the Redis service. -- [`redis_persistence`](#spec.userConfig.redis_persistence-property){: name='spec.userConfig.redis_persistence-property'} (string, Enum: `off`, `rdb`). When persistence is `rdb`, Redis does RDB dumps each 10 minutes if any key is changed. Also RDB dumps are done according to backup schedule for backup purposes. When persistence is `off`, no RDB dumps and backups are done, so data can be lost at any moment if service is restarted for any reason, or if service is powered off. Also service can't be forked. +- [`redis_persistence`](#spec.userConfig.redis_persistence-property){: name='spec.userConfig.redis_persistence-property'} (string, Enum: `off`, `rdb`). When persistence is `rdb`, Redis does RDB dumps each 10 minutes if any key is changed. Also RDB dumps are done according to the backup schedule for backup purposes. When persistence is `off`, no RDB dumps or backups are done, so data can be lost at any moment if the service is restarted for any reason, or if the service is powered off. Also, the service can't be forked. - [`redis_pubsub_client_output_buffer_limit`](#spec.userConfig.redis_pubsub_client_output_buffer_limit-property){: name='spec.userConfig.redis_pubsub_client_output_buffer_limit-property'} (integer, Minimum: 32, Maximum: 512). Set output buffer limit for pub / sub clients in MB. The value is the hard limit, the soft limit is 1/4 of the hard limit. When setting the limit, be mindful of the available memory in the selected service plan. - [`redis_ssl`](#spec.userConfig.redis_ssl-property){: name='spec.userConfig.redis_ssl-property'} (boolean). Require SSL to access Redis. - [`redis_timeout`](#spec.userConfig.redis_timeout-property){: name='spec.userConfig.redis_timeout-property'} (integer, Minimum: 0, Maximum: 31536000). Redis idle connection timeout in seconds. diff --git a/docs/docs/api-reference/serviceintegration.md b/docs/docs/api-reference/serviceintegration.md index bdc7bc22..457b4758 100644 --- a/docs/docs/api-reference/serviceintegration.md +++ b/docs/docs/api-reference/serviceintegration.md @@ -279,7 +279,7 @@ Tables to create. **Required** - [`columns`](#spec.clickhouseKafka.tables.columns-property){: name='spec.clickhouseKafka.tables.columns-property'} (array of objects, MaxItems: 100). Table columns. See below for [nested schema](#spec.clickhouseKafka.tables.columns). -- [`data_format`](#spec.clickhouseKafka.tables.data_format-property){: name='spec.clickhouseKafka.tables.data_format-property'} (string, Enum: `Avro`, `CSV`, `JSONAsString`, `JSONCompactEachRow`, `JSONCompactStringsEachRow`, `JSONEachRow`, `JSONStringsEachRow`, `MsgPack`, `TSKV`, `TSV`, `TabSeparated`, `RawBLOB`, `AvroConfluent`). Message data format. +- [`data_format`](#spec.clickhouseKafka.tables.data_format-property){: name='spec.clickhouseKafka.tables.data_format-property'} (string, Enum: `Avro`, `CSV`, `JSONAsString`, `JSONCompactEachRow`, `JSONCompactStringsEachRow`, `JSONEachRow`, `JSONStringsEachRow`, `MsgPack`, `TSKV`, `TSV`, `TabSeparated`, `RawBLOB`, `AvroConfluent`, `Parquet`). Message data format. - [`group_name`](#spec.clickhouseKafka.tables.group_name-property){: name='spec.clickhouseKafka.tables.group_name-property'} (string, MinLength: 1, MaxLength: 249). Kafka consumers group. - [`name`](#spec.clickhouseKafka.tables.name-property){: name='spec.clickhouseKafka.tables.name-property'} (string, MinLength: 1, MaxLength: 40). Name of the table. - [`topics`](#spec.clickhouseKafka.tables.topics-property){: name='spec.clickhouseKafka.tables.topics-property'} (array of objects, MaxItems: 100). Kafka topics. See below for [nested schema](#spec.clickhouseKafka.tables.topics). @@ -354,6 +354,7 @@ Datadog specific user configuration options. - [`include_topics`](#spec.datadog.include_topics-property){: name='spec.datadog.include_topics-property'} (array of strings, MaxItems: 1024). List of topics to include. - [`kafka_custom_metrics`](#spec.datadog.kafka_custom_metrics-property){: name='spec.datadog.kafka_custom_metrics-property'} (array of strings, MaxItems: 1024). List of custom metrics. - [`max_jmx_metrics`](#spec.datadog.max_jmx_metrics-property){: name='spec.datadog.max_jmx_metrics-property'} (integer, Minimum: 10, Maximum: 100000). Maximum number of JMX metrics to send. +- [`mirrormaker_custom_metrics`](#spec.datadog.mirrormaker_custom_metrics-property){: name='spec.datadog.mirrormaker_custom_metrics-property'} (array of strings, MaxItems: 1024). List of custom metrics. - [`opensearch`](#spec.datadog.opensearch-property){: name='spec.datadog.opensearch-property'} (object). Datadog Opensearch Options. See below for [nested schema](#spec.datadog.opensearch). - [`redis`](#spec.datadog.redis-property){: name='spec.datadog.redis-property'} (object). Datadog Redis Options. See below for [nested schema](#spec.datadog.redis). @@ -483,7 +484,9 @@ Kafka MirrorMaker configuration values. **Optional** +- [`consumer_auto_offset_reset`](#spec.kafkaMirrormaker.kafka_mirrormaker.consumer_auto_offset_reset-property){: name='spec.kafkaMirrormaker.kafka_mirrormaker.consumer_auto_offset_reset-property'} (string, Enum: `earliest`, `latest`). Set where consumer starts to consume data. Value `earliest`: Start replication from the earliest offset. Value `latest`: Start replication from the latest offset. Default is `earliest`. - [`consumer_fetch_min_bytes`](#spec.kafkaMirrormaker.kafka_mirrormaker.consumer_fetch_min_bytes-property){: name='spec.kafkaMirrormaker.kafka_mirrormaker.consumer_fetch_min_bytes-property'} (integer, Minimum: 1, Maximum: 5242880). The minimum amount of data the server should return for a fetch request. +- [`consumer_max_poll_records`](#spec.kafkaMirrormaker.kafka_mirrormaker.consumer_max_poll_records-property){: name='spec.kafkaMirrormaker.kafka_mirrormaker.consumer_max_poll_records-property'} (integer, Minimum: 100, Maximum: 20000). Set consumer max.poll.records. The default is 500. - [`producer_batch_size`](#spec.kafkaMirrormaker.kafka_mirrormaker.producer_batch_size-property){: name='spec.kafkaMirrormaker.kafka_mirrormaker.producer_batch_size-property'} (integer, Minimum: 0, Maximum: 5242880). The batch size in bytes producer will attempt to collect before publishing to broker. - [`producer_buffer_memory`](#spec.kafkaMirrormaker.kafka_mirrormaker.producer_buffer_memory-property){: name='spec.kafkaMirrormaker.kafka_mirrormaker.producer_buffer_memory-property'} (integer, Minimum: 5242880, Maximum: 134217728). The amount of bytes producer can use for buffering data before publishing to broker. - [`producer_compression_type`](#spec.kafkaMirrormaker.kafka_mirrormaker.producer_compression_type-property){: name='spec.kafkaMirrormaker.kafka_mirrormaker.producer_compression_type-property'} (string, Enum: `gzip`, `snappy`, `lz4`, `zstd`, `none`). Specify the default compression type for producers. This configuration accepts the standard compression codecs (`gzip`, `snappy`, `lz4`, `zstd`). It additionally accepts `none` which is the default and equivalent to no compression. diff --git a/docs/docs/api-reference/serviceintegrationendpoint.md b/docs/docs/api-reference/serviceintegrationendpoint.md index 2c1628b3..025b5445 100644 --- a/docs/docs/api-reference/serviceintegrationendpoint.md +++ b/docs/docs/api-reference/serviceintegrationendpoint.md @@ -14,11 +14,11 @@ title: "ServiceIntegrationEndpoint" authSecretRef: name: aiven-token key: token - + project: aiven-project-name endpointName: my-external-postgresql endpointType: external_postgresql - + externalPostgresql: username: username password: password @@ -37,11 +37,11 @@ title: "ServiceIntegrationEndpoint" authSecretRef: name: aiven-token key: token - + project: aiven-project-name endpointName: my-external-schema-registry endpointType: external_schema_registry - + externalSchemaRegistry: url: https://schema-registry.example.com:8081 authentication: basic @@ -258,7 +258,7 @@ ExternalPostgresql configuration values. - [`password`](#spec.externalPostgresql.password-property){: name='spec.externalPostgresql.password-property'} (string, MaxLength: 256). Password. - [`ssl_client_certificate`](#spec.externalPostgresql.ssl_client_certificate-property){: name='spec.externalPostgresql.ssl_client_certificate-property'} (string, MaxLength: 16384). Client certificate. - [`ssl_client_key`](#spec.externalPostgresql.ssl_client_key-property){: name='spec.externalPostgresql.ssl_client_key-property'} (string, MaxLength: 16384). Client key. -- [`ssl_mode`](#spec.externalPostgresql.ssl_mode-property){: name='spec.externalPostgresql.ssl_mode-property'} (string, Enum: `disable`, `allow`, `prefer`, `require`, `verify-ca`, `verify-full`). SSL Mode. +- [`ssl_mode`](#spec.externalPostgresql.ssl_mode-property){: name='spec.externalPostgresql.ssl_mode-property'} (string, Enum: `require`, `verify-ca`, `verify-full`). SSL mode to use for the connection. Please note that Aiven requires TLS for all connections to external PostgreSQL services. - [`ssl_root_cert`](#spec.externalPostgresql.ssl_root_cert-property){: name='spec.externalPostgresql.ssl_root_cert-property'} (string, MaxLength: 16384). SSL Root Cert. ## externalSchemaRegistry {: #spec.externalSchemaRegistry }