From fac1eb4e937b8f5c16e701d83eedcbbc08f0d801 Mon Sep 17 00:00:00 2001 From: Emmanuel Evbuomwan Date: Wed, 20 Nov 2024 16:12:25 +0100 Subject: [PATCH] feat: mirrormaker exactly-once delivery --- CHANGELOG.md | 1 + docs/data-sources/mirrormaker_replication_flow.md | 1 + docs/resources/mirrormaker_replication_flow.md | 1 + .../service/kafka/mirrormaker_replication_flow.go | 8 ++++++++ .../kafka/mirrormaker_replication_flow_test.go | 12 +++++++----- 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f92b2767a..2972aa40a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ nav_order: 1 Defaults to aiven. - Add `aiven_service_integration_endpoint` datasource field `datadog_user_config.extra_tags_prefix`: Extra tags prefix. Defaults to aiven. +- Add `exactly_once_delivery_enabled` field to `aiven_mirrormaker_replication_flow`: Whether to enable exactly-once message delivery for Mirrormaker replication flow ## [4.29.0] - 2024-11-14 diff --git a/docs/data-sources/mirrormaker_replication_flow.md b/docs/data-sources/mirrormaker_replication_flow.md index 13efd4408..cd8648601 100644 --- a/docs/data-sources/mirrormaker_replication_flow.md +++ b/docs/data-sources/mirrormaker_replication_flow.md @@ -37,6 +37,7 @@ data "aiven_mirrormaker_replication_flow" "f1" { - `emit_backward_heartbeats_enabled` (Boolean) Whether to emit heartbeats to the direction opposite to the flow, i.e. to the source cluster. The default value is `false`. - `emit_heartbeats_enabled` (Boolean) Whether to emit heartbeats to the target cluster. The default value is `false`. - `enable` (Boolean) Enable of disable replication flows for a service. +- `exactly_once_delivery_enabled` (Boolean) Whether to enable exactly-once message delivery. We recommend you set this to `enabled` for new replications. The default value is `false`. - `id` (String) The ID of this resource. - `offset_syncs_topic_location` (String) Offset syncs topic location. The possible values are `source` and `target`. - `replication_factor` (Number) Replication factor, `>= 1`. diff --git a/docs/resources/mirrormaker_replication_flow.md b/docs/resources/mirrormaker_replication_flow.md index 71bf00fa5..1cd7cca55 100644 --- a/docs/resources/mirrormaker_replication_flow.md +++ b/docs/resources/mirrormaker_replication_flow.md @@ -59,6 +59,7 @@ resource "aiven_mirrormaker_replication_flow" "f1" { - `config_properties_exclude` (Set of String) List of topic configuration properties and/or regular expressions to not replicate. The properties that are not replicated by default are: `follower.replication.throttled.replicas`, `leader.replication.throttled.replicas`, `message.timestamp.difference.max.ms`, `message.timestamp.type`, `unclean.leader.election.enable`, and `min.insync.replicas`. Setting this overrides the defaults. For example, to enable replication for 'min.insync.replicas' and 'unclean.leader.election.enable' set this to: ["follower\\\\.replication\\\\.throttled\\\\.replicas", "leader\\\\.replication\\\\.throttled\\\\.replicas", "message\\\\.timestamp\\\\.difference\\\\.max\\\\.ms", "message\\\\.timestamp\\\\.type"] - `emit_backward_heartbeats_enabled` (Boolean) Whether to emit heartbeats to the direction opposite to the flow, i.e. to the source cluster. The default value is `false`. - `emit_heartbeats_enabled` (Boolean) Whether to emit heartbeats to the target cluster. The default value is `false`. +- `exactly_once_delivery_enabled` (Boolean) Whether to enable exactly-once message delivery. We recommend you set this to `enabled` for new replications. The default value is `false`. - `replication_factor` (Number) Replication factor, `>= 1`. - `sync_group_offsets_enabled` (Boolean) Sync consumer group offsets. The default value is `false`. - `sync_group_offsets_interval_seconds` (Number) Frequency of consumer group offset sync. The default value is `1`. diff --git a/internal/sdkprovider/service/kafka/mirrormaker_replication_flow.go b/internal/sdkprovider/service/kafka/mirrormaker_replication_flow.go index cb85f66c9..0d3f02d2a 100644 --- a/internal/sdkprovider/service/kafka/mirrormaker_replication_flow.go +++ b/internal/sdkprovider/service/kafka/mirrormaker_replication_flow.go @@ -122,6 +122,14 @@ var aivenMirrorMakerReplicationFlowSchema = map[string]*schema.Schema{ ValidateFunc: validation.IntAtLeast(1), Description: "Replication factor, `>= 1`.", }, + "exactly_once_delivery_enabled": { + Type: schema.TypeBool, + Optional: true, + Default: false, + Description: userconfig.Desc( + "Whether to enable exactly-once message delivery. We recommend you set this to `enabled` for new replications.", + ).DefaultValue(false).Build(), + }, } func ResourceMirrorMakerReplicationFlow() *schema.Resource { diff --git a/internal/sdkprovider/service/kafka/mirrormaker_replication_flow_test.go b/internal/sdkprovider/service/kafka/mirrormaker_replication_flow_test.go index b3e183406..23fc4b94f 100644 --- a/internal/sdkprovider/service/kafka/mirrormaker_replication_flow_test.go +++ b/internal/sdkprovider/service/kafka/mirrormaker_replication_flow_test.go @@ -43,10 +43,11 @@ func TestAccAivenMirrorMakerReplicationFlow_basic(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "topics_blacklist.1", ".*\\.replica"), resource.TestCheckResourceAttr(resourceName, "topics_blacklist.2", "__.*"), resource.TestCheckResourceAttr(resourceName, "config_properties_exclude.#", "0"), + resource.TestCheckResourceAttr(resourceName, "exactly_once_delivery_enabled", "true"), ), }, { - Config: testAccMirrorMakerReplicationFlowResource(rName, ` + Config: testAccMirrorMakerReplicationFlowResource(rName, ` config_properties_exclude = [ "follower\\.replication\\.throttled\\.replicas", "leader\\.replication\\.throttled\\.replicas", @@ -226,6 +227,7 @@ resource "aiven_mirrormaker_replication_flow" "foo" { emit_heartbeats_enabled = true emit_backward_heartbeats_enabled = true offset_syncs_topic_location = "source" + exactly_once_delivery_enabled = true topics = [ ".*", @@ -241,10 +243,10 @@ resource "aiven_mirrormaker_replication_flow" "foo" { } data "aiven_mirrormaker_replication_flow" "flow" { - project = data.aiven_project.foo.project - service_name = aiven_kafka_mirrormaker.mm.service_name - source_cluster = aiven_mirrormaker_replication_flow.foo.source_cluster - target_cluster = aiven_mirrormaker_replication_flow.foo.target_cluster + project = data.aiven_project.foo.project + service_name = aiven_kafka_mirrormaker.mm.service_name + source_cluster = aiven_mirrormaker_replication_flow.foo.source_cluster + target_cluster = aiven_mirrormaker_replication_flow.foo.target_cluster depends_on = [aiven_mirrormaker_replication_flow.foo] }`, os.Getenv("AIVEN_PROJECT_NAME"), name, configExclude)