feat(aiven_service_integration): automate delivery (#1616)

CHANGELOG.md

@@ -16,6 +16,8 @@ nav_order: 1
 
 - Use new user config generator to generate service integration configs
 - Fix `aiven_kafka_schema` version update
+- Add `external_aws_cloudwatch_logs`, `external_elasticsearch_logs_user_config`, `external_opensearch_logs_user_config`,
+  `prometheus_user_config` service integration configs
 
 ## [4.14.0] - 2024-02-20
 

docs/data-sources/service_integration.md

@@ -27,7 +27,7 @@ data "aiven_service_integration" "myintegration" {
 ### Required
 
 - `destination_service_name` (String) Destination service for the integration (if any)
-- `integration_type` (String) Type of the service integration. Possible values: `alertmanager`, `cassandra_cross_service_cluster`, `clickhouse_kafka`, `clickhouse_postgresql`, `dashboard`, `datadog`, `datasource`, `external_aws_cloudwatch_logs`, `external_aws_cloudwatch_metrics`, `external_elasticsearch_logs`, `external_google_cloud_logging`, `external_opensearch_logs`, `flink`, `internal_connectivity`, `jolokia`, `kafka_connect`, `kafka_logs`, `kafka_mirrormaker`, `logs`, `m3aggregator`, `m3coordinator`, `metrics`, `opensearch_cross_cluster_replication`, `opensearch_cross_cluster_search`, `prometheus`, `read_replica`, `rsyslog`, `schema_registry_proxy`
+- `integration_type` (String) Type of the service integration. Possible values: `alertmanager`, `autoscaler`, `caching`, `cassandra_cross_service_cluster`, `clickhouse_credentials`, `clickhouse_kafka`, `clickhouse_postgresql`, `dashboard`, `datadog`, `datasource`, `external_aws_cloudwatch_logs`, `external_aws_cloudwatch_metrics`, `external_elasticsearch_logs`, `external_google_cloud_logging`, `external_opensearch_logs`, `flink`, `flink_external_bigquery`, `flink_external_kafka`, `internal_connectivity`, `jolokia`, `kafka_connect`, `kafka_logs`, `kafka_mirrormaker`, `logs`, `m3aggregator`, `m3coordinator`, `metrics`, `opensearch_cross_cluster_replication`, `opensearch_cross_cluster_search`, `prometheus`, `read_replica`, `rsyslog`, `schema_registry_proxy`, `stresstester`, `thanoscompactor`, `thanosquery`, `thanosstore`, `vector`, `vmalert`
 - `project` (String) Project the integration belongs to
 - `source_service_name` (String) Source service for the integration (if any)
 
@@ -37,14 +37,18 @@ data "aiven_service_integration" "myintegration" {
 - `clickhouse_postgresql_user_config` (List of Object) ClickhousePostgresql user configurable settings (see [below for nested schema](#nestedatt--clickhouse_postgresql_user_config))
 - `datadog_user_config` (List of Object) Datadog user configurable settings (see [below for nested schema](#nestedatt--datadog_user_config))
 - `destination_endpoint_id` (String) Destination endpoint for the integration (if any)
+- `external_aws_cloudwatch_logs_user_config` (List of Object) ExternalAwsCloudwatchLogs user configurable settings (see [below for nested schema](#nestedatt--external_aws_cloudwatch_logs_user_config))
 - `external_aws_cloudwatch_metrics_user_config` (List of Object) ExternalAwsCloudwatchMetrics user configurable settings (see [below for nested schema](#nestedatt--external_aws_cloudwatch_metrics_user_config))
+- `external_elasticsearch_logs_user_config` (List of Object) ExternalElasticsearchLogs user configurable settings (see [below for nested schema](#nestedatt--external_elasticsearch_logs_user_config))
+- `external_opensearch_logs_user_config` (List of Object) ExternalOpensearchLogs user configurable settings (see [below for nested schema](#nestedatt--external_opensearch_logs_user_config))
 - `id` (String) The ID of this resource.
 - `integration_id` (String) Service Integration Id at aiven
 - `kafka_connect_user_config` (List of Object) KafkaConnect user configurable settings (see [below for nested schema](#nestedatt--kafka_connect_user_config))
 - `kafka_logs_user_config` (List of Object) KafkaLogs user configurable settings (see [below for nested schema](#nestedatt--kafka_logs_user_config))
 - `kafka_mirrormaker_user_config` (List of Object) KafkaMirrormaker user configurable settings (see [below for nested schema](#nestedatt--kafka_mirrormaker_user_config))
 - `logs_user_config` (List of Object) Logs user configurable settings (see [below for nested schema](#nestedatt--logs_user_config))
 - `metrics_user_config` (List of Object) Metrics user configurable settings (see [below for nested schema](#nestedatt--metrics_user_config))
+- `prometheus_user_config` (List of Object) Prometheus user configurable settings (see [below for nested schema](#nestedatt--prometheus_user_config))
 - `source_endpoint_id` (String) Source endpoint for the integration (if any)
 
 <a id="nestedatt--clickhouse_kafka_user_config"></a>
@@ -153,6 +157,14 @@ Read-Only:
 
 
 
+<a id="nestedatt--external_aws_cloudwatch_logs_user_config"></a>
+### Nested Schema for `external_aws_cloudwatch_logs_user_config`
+
+Read-Only:
+
+- `selected_log_fields` (List of String)
+
+
 <a id="nestedatt--external_aws_cloudwatch_metrics_user_config"></a>
 ### Nested Schema for `external_aws_cloudwatch_metrics_user_config`
 
@@ -180,6 +192,22 @@ Read-Only:
 
 
 
+<a id="nestedatt--external_elasticsearch_logs_user_config"></a>
+### Nested Schema for `external_elasticsearch_logs_user_config`
+
+Read-Only:
+
+- `selected_log_fields` (List of String)
+
+
+<a id="nestedatt--external_opensearch_logs_user_config"></a>
+### Nested Schema for `external_opensearch_logs_user_config`
+
+Read-Only:
+
+- `selected_log_fields` (List of String)
+
+
 <a id="nestedatt--kafka_connect_user_config"></a>
 ### Nested Schema for `kafka_connect_user_config`
 
@@ -278,3 +306,40 @@ Read-Only:
 - `perf_events_statements_limit` (Number)
 - `perf_events_statements_time_limit` (Number)
 
+
+
+
+<a id="nestedatt--prometheus_user_config"></a>
+### Nested Schema for `prometheus_user_config`
+
+Read-Only:
+
+- `source_mysql` (List of Object) (see [below for nested schema](#nestedobjatt--prometheus_user_config--source_mysql))
+
+<a id="nestedobjatt--prometheus_user_config--source_mysql"></a>
+### Nested Schema for `prometheus_user_config.source_mysql`
+
+Read-Only:
+
+- `telegraf` (List of Object) (see [below for nested schema](#nestedobjatt--prometheus_user_config--source_mysql--telegraf))
+
+<a id="nestedobjatt--prometheus_user_config--source_mysql--telegraf"></a>
+### Nested Schema for `prometheus_user_config.source_mysql.telegraf`
+
+Read-Only:
+
+- `gather_event_waits` (Boolean)
+- `gather_file_events_stats` (Boolean)
+- `gather_index_io_waits` (Boolean)
+- `gather_info_schema_auto_inc` (Boolean)
+- `gather_innodb_metrics` (Boolean)
+- `gather_perf_events_statements` (Boolean)
+- `gather_process_list` (Boolean)
+- `gather_slave_status` (Boolean)
+- `gather_table_io_waits` (Boolean)
+- `gather_table_lock_waits` (Boolean)
+- `gather_table_schema` (Boolean)
+- `perf_events_statements_digest_text_limit` (Number)
+- `perf_events_statements_limit` (Number)
+- `perf_events_statements_time_limit` (Number)
+

docs/resources/service_integration.md

@@ -28,7 +28,7 @@ resource "aiven_service_integration" "my_integration_metrics" {
 
 ### Required
 
-- `integration_type` (String) Type of the service integration. Possible values: `alertmanager`, `cassandra_cross_service_cluster`, `clickhouse_kafka`, `clickhouse_postgresql`, `dashboard`, `datadog`, `datasource`, `external_aws_cloudwatch_logs`, `external_aws_cloudwatch_metrics`, `external_elasticsearch_logs`, `external_google_cloud_logging`, `external_opensearch_logs`, `flink`, `internal_connectivity`, `jolokia`, `kafka_connect`, `kafka_logs`, `kafka_mirrormaker`, `logs`, `m3aggregator`, `m3coordinator`, `metrics`, `opensearch_cross_cluster_replication`, `opensearch_cross_cluster_search`, `prometheus`, `read_replica`, `rsyslog`, `schema_registry_proxy`
+- `integration_type` (String) Type of the service integration. Possible values: `alertmanager`, `autoscaler`, `caching`, `cassandra_cross_service_cluster`, `clickhouse_credentials`, `clickhouse_kafka`, `clickhouse_postgresql`, `dashboard`, `datadog`, `datasource`, `external_aws_cloudwatch_logs`, `external_aws_cloudwatch_metrics`, `external_elasticsearch_logs`, `external_google_cloud_logging`, `external_opensearch_logs`, `flink`, `flink_external_bigquery`, `flink_external_kafka`, `internal_connectivity`, `jolokia`, `kafka_connect`, `kafka_logs`, `kafka_mirrormaker`, `logs`, `m3aggregator`, `m3coordinator`, `metrics`, `opensearch_cross_cluster_replication`, `opensearch_cross_cluster_search`, `prometheus`, `read_replica`, `rsyslog`, `schema_registry_proxy`, `stresstester`, `thanoscompactor`, `thanosquery`, `thanosstore`, `vector`, `vmalert`
 - `project` (String) Project the integration belongs to
 
 ### Optional
@@ -38,12 +38,16 @@ resource "aiven_service_integration" "my_integration_metrics" {
 - `datadog_user_config` (Block List, Max: 1) Datadog user configurable settings (see [below for nested schema](#nestedblock--datadog_user_config))
 - `destination_endpoint_id` (String) Destination endpoint for the integration (if any)
 - `destination_service_name` (String) Destination service for the integration (if any)
+- `external_aws_cloudwatch_logs_user_config` (Block List, Max: 1) ExternalAwsCloudwatchLogs user configurable settings (see [below for nested schema](#nestedblock--external_aws_cloudwatch_logs_user_config))
 - `external_aws_cloudwatch_metrics_user_config` (Block List, Max: 1) ExternalAwsCloudwatchMetrics user configurable settings (see [below for nested schema](#nestedblock--external_aws_cloudwatch_metrics_user_config))
+- `external_elasticsearch_logs_user_config` (Block List, Max: 1) ExternalElasticsearchLogs user configurable settings (see [below for nested schema](#nestedblock--external_elasticsearch_logs_user_config))
+- `external_opensearch_logs_user_config` (Block List, Max: 1) ExternalOpensearchLogs user configurable settings (see [below for nested schema](#nestedblock--external_opensearch_logs_user_config))
 - `kafka_connect_user_config` (Block List, Max: 1) KafkaConnect user configurable settings (see [below for nested schema](#nestedblock--kafka_connect_user_config))
 - `kafka_logs_user_config` (Block List, Max: 1) KafkaLogs user configurable settings (see [below for nested schema](#nestedblock--kafka_logs_user_config))
 - `kafka_mirrormaker_user_config` (Block List, Max: 1) KafkaMirrormaker user configurable settings (see [below for nested schema](#nestedblock--kafka_mirrormaker_user_config))
 - `logs_user_config` (Block List, Max: 1) Logs user configurable settings (see [below for nested schema](#nestedblock--logs_user_config))
 - `metrics_user_config` (Block List, Max: 1) Metrics user configurable settings (see [below for nested schema](#nestedblock--metrics_user_config))
+- `prometheus_user_config` (Block List, Max: 1) Prometheus user configurable settings (see [below for nested schema](#nestedblock--prometheus_user_config))
 - `source_endpoint_id` (String) Source endpoint for the integration (if any)
 - `source_service_name` (String) Source service for the integration (if any)
 - `timeouts` (Block, Optional) (see [below for nested schema](#nestedblock--timeouts))
@@ -165,6 +169,14 @@ Optional:
 
 
 
+<a id="nestedblock--external_aws_cloudwatch_logs_user_config"></a>
+### Nested Schema for `external_aws_cloudwatch_logs_user_config`
+
+Optional:
+
+- `selected_log_fields` (List of String) The list of logging fields that will be sent to the integration logging service. The MESSAGE and timestamp fields are always sent.
+
+
 <a id="nestedblock--external_aws_cloudwatch_metrics_user_config"></a>
 ### Nested Schema for `external_aws_cloudwatch_metrics_user_config`
 
@@ -192,6 +204,22 @@ Required:
 
 
 
+<a id="nestedblock--external_elasticsearch_logs_user_config"></a>
+### Nested Schema for `external_elasticsearch_logs_user_config`
+
+Optional:
+
+- `selected_log_fields` (List of String) The list of logging fields that will be sent to the integration logging service. The MESSAGE and timestamp fields are always sent.
+
+
+<a id="nestedblock--external_opensearch_logs_user_config"></a>
+### Nested Schema for `external_opensearch_logs_user_config`
+
+Optional:
+
+- `selected_log_fields` (List of String) The list of logging fields that will be sent to the integration logging service. The MESSAGE and timestamp fields are always sent.
+
+
 <a id="nestedblock--kafka_connect_user_config"></a>
 ### Nested Schema for `kafka_connect_user_config`
 
@@ -296,6 +324,43 @@ Optional:
 
 
 
+<a id="nestedblock--prometheus_user_config"></a>
+### Nested Schema for `prometheus_user_config`
+
+Optional:
+
+- `source_mysql` (Block List, Max: 1) Configuration options for metrics where source service is MySQL (see [below for nested schema](#nestedblock--prometheus_user_config--source_mysql))
+
+<a id="nestedblock--prometheus_user_config--source_mysql"></a>
+### Nested Schema for `prometheus_user_config.source_mysql`
+
+Optional:
+
+- `telegraf` (Block List, Max: 1) Configuration options for Telegraf MySQL input plugin (see [below for nested schema](#nestedblock--prometheus_user_config--source_mysql--telegraf))
+
+<a id="nestedblock--prometheus_user_config--source_mysql--telegraf"></a>
+### Nested Schema for `prometheus_user_config.source_mysql.telegraf`
+
+Optional:
+
+- `gather_event_waits` (Boolean) Gather metrics from PERFORMANCE_SCHEMA.EVENT_WAITS.
+- `gather_file_events_stats` (Boolean) gather metrics from PERFORMANCE_SCHEMA.FILE_SUMMARY_BY_EVENT_NAME.
+- `gather_index_io_waits` (Boolean) Gather metrics from PERFORMANCE_SCHEMA.TABLE_IO_WAITS_SUMMARY_BY_INDEX_USAGE.
+- `gather_info_schema_auto_inc` (Boolean) Gather auto_increment columns and max values from information schema.
+- `gather_innodb_metrics` (Boolean) Gather metrics from INFORMATION_SCHEMA.INNODB_METRICS.
+- `gather_perf_events_statements` (Boolean) Gather metrics from PERFORMANCE_SCHEMA.EVENTS_STATEMENTS_SUMMARY_BY_DIGEST.
+- `gather_process_list` (Boolean) Gather thread state counts from INFORMATION_SCHEMA.PROCESSLIST.
+- `gather_slave_status` (Boolean) Gather metrics from SHOW SLAVE STATUS command output.
+- `gather_table_io_waits` (Boolean) Gather metrics from PERFORMANCE_SCHEMA.TABLE_IO_WAITS_SUMMARY_BY_TABLE.
+- `gather_table_lock_waits` (Boolean) Gather metrics from PERFORMANCE_SCHEMA.TABLE_LOCK_WAITS.
+- `gather_table_schema` (Boolean) Gather metrics from INFORMATION_SCHEMA.TABLES.
+- `perf_events_statements_digest_text_limit` (Number) Truncates digest text from perf_events_statements into this many characters.
+- `perf_events_statements_limit` (Number) Limits metrics from perf_events_statements.
+- `perf_events_statements_time_limit` (Number) Only include perf_events_statements whose last seen is less than this many seconds.
+
+
+
+
 <a id="nestedblock--timeouts"></a>
 ### Nested Schema for `timeouts`
 

go.mod

@@ -4,6 +4,7 @@ go 1.22
 
 require (
 	github.com/aiven/aiven-go-client/v2 v2.12.0
+	github.com/aiven/go-client-codegen v0.2.0
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/dave/jennifer v1.7.0
 	github.com/docker/go-units v0.5.0
@@ -19,8 +20,8 @@ require (
 	github.com/samber/lo v1.39.0
 	github.com/stoewer/go-strcase v1.3.0
 	github.com/stretchr/testify v1.8.4
-	golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819
-	golang.org/x/tools v0.13.0
+	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3
+	golang.org/x/tools v0.17.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -37,7 +38,7 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.7.5 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
-	golang.org/x/sync v0.5.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect
@@ -81,7 +82,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/klauspost/compress v1.15.11 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -98,7 +99,7 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	golang.org/x/crypto v0.18.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.18.0 // indirect
+	golang.org/x/net v0.20.0 // indirect
 	golang.org/x/oauth2 v0.14.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect