From 34707cdf7319232aa0cde4a00ef16a0e077eccc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Warzy=C5=84ski?= Date: Sat, 22 Apr 2023 16:30:53 +0200 Subject: [PATCH 01/13] test: filter_aws ec2 tags fail should not prevent other data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Modify existing test case when filter aws receives 500 status code from the tag keys listing HTTP endpoint. Add requirement for the filter aws to inject other AWS datapoints even if EC2 tags fetch fails. Implementation is going to be followed in the next commits. (Unit test fails on this state.) Signed-off-by: Mateusz Warzyński --- tests/runtime/filter_aws.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/runtime/filter_aws.c b/tests/runtime/filter_aws.c index 770915a01ba..2d682d3256d 100644 --- a/tests/runtime/filter_aws.c +++ b/tests/runtime/filter_aws.c @@ -287,6 +287,13 @@ void flb_test_aws_ec2_tags_list_500() { char *result; request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/instance-id/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "i-0e66fc7f9809d7168"), + set(PAYLOAD_SIZE, 19) + ), response( expect(URI, "/latest/meta-data/tags/instance"), expect(METHOD, FLB_HTTP_GET), @@ -332,7 +339,7 @@ void flb_test_aws_ec2_tags_list_500() { TEST_CHECK(filter_ffd >= 0); ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); TEST_CHECK(ret == 0); - ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "true", NULL); TEST_CHECK(ret == 0); ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); TEST_CHECK(ret == 0); @@ -355,6 +362,12 @@ void flb_test_aws_ec2_tags_list_500() { if (!TEST_CHECK(result != NULL)) { TEST_MSG("output:%s\n", output); } + result = strstr(output, "i-0e66fc7f9809d7168"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("ec2 instance id is not present in the output log\n"); + TEST_MSG("however, it should be injected despite ec2 tags fetching errors\n"); + TEST_MSG("output:%s\n", output); + } } else { TEST_CHECK(false); From a9ab68cf680817a94114bf7d42135353fd7f9f0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Warzy=C5=84ski?= Date: Sat, 22 Apr 2023 16:39:25 +0200 Subject: [PATCH 02/13] filter: aws improve comment about ec2 metadata unrecoverable error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Warzyński --- plugins/filter_aws/aws.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/filter_aws/aws.c b/plugins/filter_aws/aws.c index e30d236e38d..10d86a37d27 100644 --- a/plugins/filter_aws/aws.c +++ b/plugins/filter_aws/aws.c @@ -192,7 +192,8 @@ static int cb_aws_init(struct flb_filter_instance *f_ins, if (ret < 0) { /* If the metadata fetch fails, the plugin continues to work. */ /* Every flush will attempt to fetch ec2 metadata, if needed. */ - /* In the error is unrecoverable (-3), it exits and does not retry. */ + /* If the error is unrecoverable (-3), it exits and does not retry. */ + /* e.g.: unrecoverable errors might be related to invalid configuration. */ if (ret == -3) { flb_free(ctx); return -1; From 21f74ebd1c3565ca01a5396fe4a689b1b2431a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Warzy=C5=84ski?= Date: Sun, 18 Jun 2023 03:32:32 +0200 Subject: [PATCH 03/13] filter: aws introduce metadata_group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit allows to store information about specific metadata groups in the ctx it prepares the ground for defining different options of retries based on the metadata group at the beginning, commit defines two information groups: - base (includes everything except tags), - tags, in the future, it should be quite easy to divide 'base' information group into more granular groups such that we have more control over how we want to treat fetching information (and importance) of each specific datapoint Signed-off-by: Mateusz Warzyński --- plugins/filter_aws/aws.c | 136 ++++++++++++++++++++++++++------------- plugins/filter_aws/aws.h | 27 ++++++++ 2 files changed, 117 insertions(+), 46 deletions(-) diff --git a/plugins/filter_aws/aws.c b/plugins/filter_aws/aws.c index 10d86a37d27..57a6c42deb4 100644 --- a/plugins/filter_aws/aws.c +++ b/plugins/filter_aws/aws.c @@ -54,53 +54,60 @@ static void expose_aws_meta(struct flb_filter_aws *ctx) flb_env_set(env, "aws", "enabled"); - if (ctx->availability_zone_include) { - flb_env_set(env, - "aws." FLB_FILTER_AWS_AVAILABILITY_ZONE_KEY, - ctx->availability_zone); - } + if (ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE].done && + !ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE].exposed) { + if (ctx->availability_zone_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_AVAILABILITY_ZONE_KEY, + ctx->availability_zone); + } - if (ctx->instance_id_include) { - flb_env_set(env, - "aws." FLB_FILTER_AWS_INSTANCE_ID_KEY, - ctx->instance_id); - } + if (ctx->instance_id_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_INSTANCE_ID_KEY, + ctx->instance_id); + } - if (ctx->instance_type_include) { - flb_env_set(env, - "aws." FLB_FILTER_AWS_INSTANCE_TYPE_KEY, - ctx->instance_type); - } + if (ctx->instance_type_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_INSTANCE_TYPE_KEY, + ctx->instance_type); + } - if (ctx->private_ip_include) { - flb_env_set(env, - "aws." FLB_FILTER_AWS_PRIVATE_IP_KEY, - ctx->private_ip); - } + if (ctx->private_ip_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_PRIVATE_IP_KEY, + ctx->private_ip); + } - if (ctx->vpc_id_include) { - flb_env_set(env, - "aws." FLB_FILTER_AWS_VPC_ID_KEY, - ctx->vpc_id); - } + if (ctx->vpc_id_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_VPC_ID_KEY, + ctx->vpc_id); + } - if (ctx->ami_id_include) { - flb_env_set(env, - "aws." FLB_FILTER_AWS_AMI_ID_KEY, - ctx->ami_id); - } + if (ctx->ami_id_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_AMI_ID_KEY, + ctx->ami_id); + } - if (ctx->account_id_include) { - flb_env_set(env, - "aws." FLB_FILTER_AWS_ACCOUNT_ID_KEY, - ctx->account_id); - } + if (ctx->account_id_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_ACCOUNT_ID_KEY, + ctx->account_id); + } + + if (ctx->hostname_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_HOSTNAME_KEY, + ctx->hostname); + } - if (ctx->hostname_include) { - flb_env_set(env, - "aws." FLB_FILTER_AWS_HOSTNAME_KEY, - ctx->hostname); + ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE].exposed = FLB_TRUE; } + + /* TODO: expose aws ec2 tags in flb_env_set */ } static int cb_aws_init(struct flb_filter_instance *f_ins, @@ -185,16 +192,14 @@ static int cb_aws_init(struct flb_filter_instance *f_ins, return -1; } - ctx->metadata_retrieved = FLB_FALSE; - /* Retrieve metadata */ ret = get_ec2_metadata(ctx); if (ret < 0) { /* If the metadata fetch fails, the plugin continues to work. */ /* Every flush will attempt to fetch ec2 metadata, if needed. */ - /* If the error is unrecoverable (-3), it exits and does not retry. */ + /* If the error is unrecoverable, it exits and does not retry. */ /* e.g.: unrecoverable errors might be related to invalid configuration. */ - if (ret == -3) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { flb_free(ctx); return -1; } @@ -529,7 +534,7 @@ static int get_ec2_tag_enabled(struct flb_filter_aws *ctx) if (tags_include && tags_exclude) { flb_plg_error(ctx->ins, "configuration is invalid, both tags_include" " and tags_exclude are specified at the same time"); - return -3; + return FLB_FILTER_AWS_CONFIGURATION_ERROR; } if (!tags_include && tags_exclude) { /* copy const string in order to use strtok which modifes the string */ @@ -607,12 +612,15 @@ static int get_ec2_tags(struct flb_filter_aws *ctx) /* * Makes a call to IMDS to set get the values of all metadata fields. * It can be called repeatedly if some metadata calls initially do not succeed. + * However, if function succeeds, the expectation is that it shouldn't be called again. */ -static int get_ec2_metadata(struct flb_filter_aws *ctx) +static int get_ec2_metadata_base(struct flb_filter_aws *ctx) { int ret; - int i; + /* TODO: check this during review, I added this line -- I think it makes it more correct. */ + /* it also might be fine to just delete new_keys as it's not really used anywhere */ + ctx->new_keys = 0; if (ctx->instance_id_include && !ctx->instance_id) { ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_INSTANCE_ID_PATH, &ctx->instance_id, @@ -702,6 +710,14 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx) ctx->new_keys++; } + return 0; +} + +static int get_ec2_metadata_tags(struct flb_filter_aws *ctx) +{ + int ret; + int i; + if (ctx->tags_enabled && !ctx->tags_fetched) { ret = get_ec2_tags(ctx); if (ret < 0) { @@ -715,6 +731,34 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx) } } + return 0; +} + +/* + * Fetches all metadata values, including tags, from IMDS. + */ +static int get_ec2_metadata(struct flb_filter_aws *ctx) +{ + int ret; + + if (!ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE].done) { + ret = get_ec2_metadata_base(ctx); + if (ret < 0) { + return ret; + } + ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE].done = FLB_TRUE; + } + + if (!ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].done) { + /* TODO: retries */ + ret = get_ec2_metadata_tags(ctx); + + if (ret < 0) { + return ret; + } + ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].done = FLB_TRUE; + } + ctx->metadata_retrieved = FLB_TRUE; return 0; } diff --git a/plugins/filter_aws/aws.h b/plugins/filter_aws/aws.h index fa6843ceeb7..6305fdf4bae 100644 --- a/plugins/filter_aws/aws.h +++ b/plugins/filter_aws/aws.h @@ -40,6 +40,30 @@ #define FLB_FILTER_AWS_HOSTNAME_KEY "hostname" #define FLB_FILTER_AWS_HOSTNAME_KEY_LEN 8 +/* defines returned value for cases when configuration is invalid and program should exit */ +#define FLB_FILTER_AWS_CONFIGURATION_ERROR -100 + +/* defines a group of information, is used as an index in the ctx->metadata_groups + * to be specific, it is an index at array flb_filter_aws->metadata_groups */ +#define FLB_FILTER_AWS_METADATA_GROUP_BASE 0 +#define FLB_FILTER_AWS_METADATA_GROUP_TAGS 1 + +#define FLB_FILTER_AWS_METADATA_GROUP_NUM 2 /* used to define required memory */ + +struct flb_filter_aws_metadata_group { + /* defines if fetch function for the information group was already done successfully + * if set to FLB_FALSE after first attempt, then most likely another retry will be + * required + * done set to FLB_TRUE does not mean that information was retrieved, as it might + * be disabled */ + int done; + /* defines if information was already exposed in the filter for envs */ + int exposed; + + // TODO: possibly it will need new_keys or something related to injecting into the msgpack +}; + + struct flb_filter_aws { struct flb_filter_aws_init_options *options; @@ -115,6 +139,9 @@ struct flb_filter_aws { /* number of new keys added by this plugin */ int new_keys; + /* metadata groups contains information for potential retries and + * if group was already fetched successfully */ + struct flb_filter_aws_metadata_group metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_NUM]; int metadata_retrieved; /* Plugin can use EC2 metadata v1 or v2; default is v2 */ From 9155583f50b9769bad3f798cff14cdb828a55f85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Warzy=C5=84ski?= Date: Sun, 18 Jun 2023 03:53:43 +0200 Subject: [PATCH 04/13] filter: aws tags are not critical MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit allows to inject base metadata group into the logs even if tags were not fetched in this version fetching tags will be retried indefinitely with every flush, where each retry is going to produce error logs Signed-off-by: Mateusz Warzyński --- plugins/filter_aws/aws.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/plugins/filter_aws/aws.c b/plugins/filter_aws/aws.c index 57a6c42deb4..2ed457f01c4 100644 --- a/plugins/filter_aws/aws.c +++ b/plugins/filter_aws/aws.c @@ -740,6 +740,7 @@ static int get_ec2_metadata_tags(struct flb_filter_aws *ctx) static int get_ec2_metadata(struct flb_filter_aws *ctx) { int ret; + int failures = 0; if (!ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE].done) { ret = get_ec2_metadata_base(ctx); @@ -750,16 +751,23 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx) } if (!ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].done) { - /* TODO: retries */ + /* TODO: retries with Fixed Interval Rate */ + /* at the moment it will retry with every flush */ ret = get_ec2_metadata_tags(ctx); - - if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { return ret; } - ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].done = FLB_TRUE; + if (ret == 0) { + ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].done = FLB_TRUE; + } else { + failures++; + } + } + + if (failures == 0) { + ctx->metadata_retrieved = FLB_TRUE; } - ctx->metadata_retrieved = FLB_TRUE; return 0; } From 0ba5899e92810b1f873f9c75b55c7ffef5b8404e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Warzy=C5=84ski?= Date: Sun, 18 Jun 2023 04:23:08 +0200 Subject: [PATCH 05/13] filter: aws tags fetch retries at >5s interval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit retry interval is configurable with 'tags_retry_interval_s' option which accepts integer, which allows the user to override the default 5s Signed-off-by: Mateusz Warzyński --- plugins/filter_aws/aws.c | 56 +++++++++++++++++++++++++++++++++------- plugins/filter_aws/aws.h | 7 ++++- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/plugins/filter_aws/aws.c b/plugins/filter_aws/aws.c index 2ed457f01c4..fcde5c1f13c 100644 --- a/plugins/filter_aws/aws.c +++ b/plugins/filter_aws/aws.c @@ -618,6 +618,8 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) { int ret; + ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE].last_execution = time(NULL); + /* TODO: check this during review, I added this line -- I think it makes it more correct. */ /* it also might be fine to just delete new_keys as it's not really used anywhere */ ctx->new_keys = 0; @@ -718,6 +720,8 @@ static int get_ec2_metadata_tags(struct flb_filter_aws *ctx) int ret; int i; + ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].last_execution = time(NULL); + if (ctx->tags_enabled && !ctx->tags_fetched) { ret = get_ec2_tags(ctx); if (ret < 0) { @@ -734,13 +738,37 @@ static int get_ec2_metadata_tags(struct flb_filter_aws *ctx) return 0; } +static int ec2_metadata_group_should_fetch(struct flb_filter_aws *ctx, int group) +{ + time_t now, required_interval, interval; + + required_interval = ctx->metadata_groups[group].retry_required_interval; + if (required_interval == 0) { + return FLB_TRUE; + } + + now = time(NULL); + + interval = now - ctx->metadata_groups[group].last_execution; + + if (interval < required_interval) { + return FLB_FALSE; + } + return FLB_TRUE; +} + /* * Fetches all metadata values, including tags, from IMDS. + * Function handles retries as configured for each metadata group. + * + * Returns 0 on success, negative values on failures. + * Returns FLB_FILTER_AWS_CONFIGURATION_ERROR in case of configuration error. */ static int get_ec2_metadata(struct flb_filter_aws *ctx) { int ret; int failures = 0; + int fetches_skipped = 0; if (!ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE].done) { ret = get_ec2_metadata_base(ctx); @@ -751,20 +779,22 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx) } if (!ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].done) { - /* TODO: retries with Fixed Interval Rate */ - /* at the moment it will retry with every flush */ - ret = get_ec2_metadata_tags(ctx); - if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { - return ret; - } - if (ret == 0) { - ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].done = FLB_TRUE; + if (!ec2_metadata_group_should_fetch(ctx, FLB_FILTER_AWS_METADATA_GROUP_TAGS)) { + fetches_skipped++; } else { - failures++; + ret = get_ec2_metadata_tags(ctx); + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + if (ret == 0) { + ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].done = FLB_TRUE; + } else { + failures++; + } } } - if (failures == 0) { + if (failures == 0 && fetches_skipped == 0) { ctx->metadata_retrieved = FLB_TRUE; } @@ -1101,6 +1131,12 @@ static struct flb_config_map config_map[] = { "if both tags_include and tags_exclude are specified, configuration is invalid" " and plugin fails" }, + { + FLB_CONFIG_MAP_INT, "tags_retry_interval_s", "5", + 0, FLB_TRUE, offsetof(struct flb_filter_aws, + metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].retry_required_interval), + "Defines minimum duration between retries for fetching EC2 instance tags" + }, {0} }; diff --git a/plugins/filter_aws/aws.h b/plugins/filter_aws/aws.h index 6305fdf4bae..f963645db9f 100644 --- a/plugins/filter_aws/aws.h +++ b/plugins/filter_aws/aws.h @@ -60,7 +60,12 @@ struct flb_filter_aws_metadata_group { /* defines if information was already exposed in the filter for envs */ int exposed; - // TODO: possibly it will need new_keys or something related to injecting into the msgpack + /* defines a timestamp of last execution of fetch method related to the group */ + /* unit: seconds */ + time_t last_execution; + /* defines a minimal interval before consecutive retries */ + /* unit: seconds */ + time_t retry_required_interval; }; From ca53d691837c4cfb5032f7e2e5979ea6ffe89c1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Warzy=C5=84ski?= Date: Sun, 18 Jun 2023 04:36:34 +0200 Subject: [PATCH 06/13] filter: aws drop unused 'new_keys' from ctx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Warzyński --- plugins/filter_aws/aws.c | 17 ----------------- plugins/filter_aws/aws.h | 3 --- 2 files changed, 20 deletions(-) diff --git a/plugins/filter_aws/aws.c b/plugins/filter_aws/aws.c index fcde5c1f13c..ac0e6ef8dd8 100644 --- a/plugins/filter_aws/aws.c +++ b/plugins/filter_aws/aws.c @@ -620,9 +620,6 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE].last_execution = time(NULL); - /* TODO: check this during review, I added this line -- I think it makes it more correct. */ - /* it also might be fine to just delete new_keys as it's not really used anywhere */ - ctx->new_keys = 0; if (ctx->instance_id_include && !ctx->instance_id) { ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_INSTANCE_ID_PATH, &ctx->instance_id, @@ -631,7 +628,6 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) flb_plg_error(ctx->ins, "Failed to get instance ID"); return -1; } - ctx->new_keys++; } if (ctx->availability_zone_include && !ctx->availability_zone) { @@ -643,7 +639,6 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) flb_plg_error(ctx->ins, "Failed to get instance AZ"); return -1; } - ctx->new_keys++; } if (ctx->instance_type_include && !ctx->instance_type) { @@ -654,7 +649,6 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) flb_plg_error(ctx->ins, "Failed to get instance type"); return -1; } - ctx->new_keys++; } if (ctx->private_ip_include && !ctx->private_ip) { @@ -665,7 +659,6 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) flb_plg_error(ctx->ins, "Failed to get instance private IP"); return -1; } - ctx->new_keys++; } if (ctx->vpc_id_include && !ctx->vpc_id) { @@ -675,7 +668,6 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) flb_plg_error(ctx->ins, "Failed to get instance VPC ID"); return -1; } - ctx->new_keys++; } if (ctx->ami_id_include && !ctx->ami_id) { @@ -686,7 +678,6 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) flb_plg_error(ctx->ins, "Failed to get AMI ID"); return -1; } - ctx->new_keys++; } if (ctx->account_id_include && !ctx->account_id) { @@ -698,7 +689,6 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) flb_plg_error(ctx->ins, "Failed to get Account ID"); return -1; } - ctx->new_keys++; } if (ctx->hostname_include && !ctx->hostname) { @@ -709,7 +699,6 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) flb_plg_error(ctx->ins, "Failed to get Hostname"); return -1; } - ctx->new_keys++; } return 0; @@ -718,7 +707,6 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) static int get_ec2_metadata_tags(struct flb_filter_aws *ctx) { int ret; - int i; ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].last_execution = time(NULL); @@ -728,11 +716,6 @@ static int get_ec2_metadata_tags(struct flb_filter_aws *ctx) flb_plg_error(ctx->ins, "Failed to get instance EC2 Tags"); return ret; } - for (i = 0; i < ctx->tags_count; i++) { - if (ctx->tag_is_enabled[i] == FLB_TRUE) { - ctx->new_keys++; - } - } } return 0; diff --git a/plugins/filter_aws/aws.h b/plugins/filter_aws/aws.h index f963645db9f..389bf08dc2f 100644 --- a/plugins/filter_aws/aws.h +++ b/plugins/filter_aws/aws.h @@ -141,9 +141,6 @@ struct flb_filter_aws { /* e.g.: if tag_is_enabled[0] = FALSE, then filter aws should not inject first tag */ int *tag_is_enabled; - /* number of new keys added by this plugin */ - int new_keys; - /* metadata groups contains information for potential retries and * if group was already fetched successfully */ struct flb_filter_aws_metadata_group metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_NUM]; From e3af07e7f9c9d04959f4a31a0cbce7000512f20f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Warzy=C5=84ski?= Date: Wed, 21 Jun 2023 00:53:59 +0200 Subject: [PATCH 07/13] filter: aws review fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Warzyński --- plugins/filter_aws/aws.c | 20 +++++++++++--------- plugins/filter_aws/aws.h | 8 ++++---- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/plugins/filter_aws/aws.c b/plugins/filter_aws/aws.c index ac0e6ef8dd8..cda6f92e32a 100644 --- a/plugins/filter_aws/aws.c +++ b/plugins/filter_aws/aws.c @@ -618,7 +618,8 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) { int ret; - ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE].last_execution = time(NULL); + ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE]. + last_fetch_attempt = time(NULL); if (ctx->instance_id_include && !ctx->instance_id) { ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_INSTANCE_ID_PATH, @@ -708,7 +709,8 @@ static int get_ec2_metadata_tags(struct flb_filter_aws *ctx) { int ret; - ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].last_execution = time(NULL); + ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS]. + last_fetch_attempt = time(NULL); if (ctx->tags_enabled && !ctx->tags_fetched) { ret = get_ec2_tags(ctx); @@ -732,7 +734,7 @@ static int ec2_metadata_group_should_fetch(struct flb_filter_aws *ctx, int group now = time(NULL); - interval = now - ctx->metadata_groups[group].last_execution; + interval = now - ctx->metadata_groups[group].last_fetch_attempt; if (interval < required_interval) { return FLB_FALSE; @@ -750,8 +752,8 @@ static int ec2_metadata_group_should_fetch(struct flb_filter_aws *ctx, int group static int get_ec2_metadata(struct flb_filter_aws *ctx) { int ret; - int failures = 0; - int fetches_skipped = 0; + int no_failures = FLB_TRUE; + int no_fetches_skipped = FLB_TRUE; if (!ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE].done) { ret = get_ec2_metadata_base(ctx); @@ -763,7 +765,7 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx) if (!ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].done) { if (!ec2_metadata_group_should_fetch(ctx, FLB_FILTER_AWS_METADATA_GROUP_TAGS)) { - fetches_skipped++; + no_fetches_skipped = FLB_FALSE; } else { ret = get_ec2_metadata_tags(ctx); if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { @@ -772,12 +774,12 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx) if (ret == 0) { ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].done = FLB_TRUE; } else { - failures++; + no_failures = FLB_FALSE; } } } - if (failures == 0 && fetches_skipped == 0) { + if (no_failures && no_fetches_skipped) { ctx->metadata_retrieved = FLB_TRUE; } @@ -1115,7 +1117,7 @@ static struct flb_config_map config_map[] = { " and plugin fails" }, { - FLB_CONFIG_MAP_INT, "tags_retry_interval_s", "5", + FLB_CONFIG_MAP_INT, "tags_retry_interval_s", "300", 0, FLB_TRUE, offsetof(struct flb_filter_aws, metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].retry_required_interval), "Defines minimum duration between retries for fetching EC2 instance tags" diff --git a/plugins/filter_aws/aws.h b/plugins/filter_aws/aws.h index 389bf08dc2f..187ed16a8e6 100644 --- a/plugins/filter_aws/aws.h +++ b/plugins/filter_aws/aws.h @@ -48,7 +48,7 @@ #define FLB_FILTER_AWS_METADATA_GROUP_BASE 0 #define FLB_FILTER_AWS_METADATA_GROUP_TAGS 1 -#define FLB_FILTER_AWS_METADATA_GROUP_NUM 2 /* used to define required memory */ +#define FLB_FILTER_AWS_METADATA_GROUP_LEN 2 /* used to define required memory */ struct flb_filter_aws_metadata_group { /* defines if fetch function for the information group was already done successfully @@ -61,8 +61,8 @@ struct flb_filter_aws_metadata_group { int exposed; /* defines a timestamp of last execution of fetch method related to the group */ - /* unit: seconds */ - time_t last_execution; + /* unit: timestamp in seconds */ + time_t last_fetch_attempt; /* defines a minimal interval before consecutive retries */ /* unit: seconds */ time_t retry_required_interval; @@ -143,7 +143,7 @@ struct flb_filter_aws { /* metadata groups contains information for potential retries and * if group was already fetched successfully */ - struct flb_filter_aws_metadata_group metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_NUM]; + struct flb_filter_aws_metadata_group metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_LEN]; int metadata_retrieved; /* Plugin can use EC2 metadata v1 or v2; default is v2 */ From 73928b47eb74167b9b54b0c299127eba42caca30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Warzy=C5=84ski?= Date: Wed, 21 Jun 2023 01:02:29 +0200 Subject: [PATCH 08/13] filter: aws retry config rename to 'retry_interval_s' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Warzyński --- plugins/filter_aws/aws.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/filter_aws/aws.c b/plugins/filter_aws/aws.c index cda6f92e32a..f952177285c 100644 --- a/plugins/filter_aws/aws.c +++ b/plugins/filter_aws/aws.c @@ -1117,7 +1117,9 @@ static struct flb_config_map config_map[] = { " and plugin fails" }, { - FLB_CONFIG_MAP_INT, "tags_retry_interval_s", "300", + FLB_CONFIG_MAP_INT, "retry_interval_s", "300", + /* for the time being, the only group which has retries is ec2 tags */ + /* therefore configuration immediately sets this value just for ec2 tags */ 0, FLB_TRUE, offsetof(struct flb_filter_aws, metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].retry_required_interval), "Defines minimum duration between retries for fetching EC2 instance tags" From 668cf72946901ced45e09da90e0ff438deb9b2bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Warzy=C5=84ski?= Date: Sat, 24 Jun 2023 15:29:54 +0200 Subject: [PATCH 09/13] filter: aws ctx define groups as attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit group_tag and group_base instead of array of groups Signed-off-by: Mateusz Warzyński --- plugins/filter_aws/aws.c | 28 ++++++++++++++-------------- plugins/filter_aws/aws.h | 12 +++--------- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/plugins/filter_aws/aws.c b/plugins/filter_aws/aws.c index f952177285c..eccb1c20444 100644 --- a/plugins/filter_aws/aws.c +++ b/plugins/filter_aws/aws.c @@ -54,8 +54,8 @@ static void expose_aws_meta(struct flb_filter_aws *ctx) flb_env_set(env, "aws", "enabled"); - if (ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE].done && - !ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE].exposed) { + if (ctx->group_base.done && + !ctx->group_base.exposed) { if (ctx->availability_zone_include) { flb_env_set(env, "aws." FLB_FILTER_AWS_AVAILABILITY_ZONE_KEY, @@ -104,7 +104,7 @@ static void expose_aws_meta(struct flb_filter_aws *ctx) ctx->hostname); } - ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE].exposed = FLB_TRUE; + ctx->group_base.exposed = FLB_TRUE; } /* TODO: expose aws ec2 tags in flb_env_set */ @@ -618,7 +618,7 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) { int ret; - ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE]. + ctx->group_base. last_fetch_attempt = time(NULL); if (ctx->instance_id_include && !ctx->instance_id) { @@ -709,7 +709,7 @@ static int get_ec2_metadata_tags(struct flb_filter_aws *ctx) { int ret; - ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS]. + ctx->group_tag. last_fetch_attempt = time(NULL); if (ctx->tags_enabled && !ctx->tags_fetched) { @@ -723,18 +723,18 @@ static int get_ec2_metadata_tags(struct flb_filter_aws *ctx) return 0; } -static int ec2_metadata_group_should_fetch(struct flb_filter_aws *ctx, int group) +static int ec2_metadata_group_should_fetch(struct flb_filter_aws_metadata_group *group) { time_t now, required_interval, interval; - required_interval = ctx->metadata_groups[group].retry_required_interval; + required_interval = group->retry_required_interval; if (required_interval == 0) { return FLB_TRUE; } now = time(NULL); - interval = now - ctx->metadata_groups[group].last_fetch_attempt; + interval = now - group->last_fetch_attempt; if (interval < required_interval) { return FLB_FALSE; @@ -755,16 +755,16 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx) int no_failures = FLB_TRUE; int no_fetches_skipped = FLB_TRUE; - if (!ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE].done) { + if (!ctx->group_base.done) { ret = get_ec2_metadata_base(ctx); if (ret < 0) { return ret; } - ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_BASE].done = FLB_TRUE; + ctx->group_base.done = FLB_TRUE; } - if (!ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].done) { - if (!ec2_metadata_group_should_fetch(ctx, FLB_FILTER_AWS_METADATA_GROUP_TAGS)) { + if (!ctx->group_tag.done) { + if (!ec2_metadata_group_should_fetch(&ctx->group_tag)) { no_fetches_skipped = FLB_FALSE; } else { ret = get_ec2_metadata_tags(ctx); @@ -772,7 +772,7 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx) return ret; } if (ret == 0) { - ctx->metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].done = FLB_TRUE; + ctx->group_tag.done = FLB_TRUE; } else { no_failures = FLB_FALSE; } @@ -1121,7 +1121,7 @@ static struct flb_config_map config_map[] = { /* for the time being, the only group which has retries is ec2 tags */ /* therefore configuration immediately sets this value just for ec2 tags */ 0, FLB_TRUE, offsetof(struct flb_filter_aws, - metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_TAGS].retry_required_interval), + group_tag.retry_required_interval), "Defines minimum duration between retries for fetching EC2 instance tags" }, {0} diff --git a/plugins/filter_aws/aws.h b/plugins/filter_aws/aws.h index 187ed16a8e6..c6458388f73 100644 --- a/plugins/filter_aws/aws.h +++ b/plugins/filter_aws/aws.h @@ -43,13 +43,6 @@ /* defines returned value for cases when configuration is invalid and program should exit */ #define FLB_FILTER_AWS_CONFIGURATION_ERROR -100 -/* defines a group of information, is used as an index in the ctx->metadata_groups - * to be specific, it is an index at array flb_filter_aws->metadata_groups */ -#define FLB_FILTER_AWS_METADATA_GROUP_BASE 0 -#define FLB_FILTER_AWS_METADATA_GROUP_TAGS 1 - -#define FLB_FILTER_AWS_METADATA_GROUP_LEN 2 /* used to define required memory */ - struct flb_filter_aws_metadata_group { /* defines if fetch function for the information group was already done successfully * if set to FLB_FALSE after first attempt, then most likely another retry will be @@ -141,9 +134,10 @@ struct flb_filter_aws { /* e.g.: if tag_is_enabled[0] = FALSE, then filter aws should not inject first tag */ int *tag_is_enabled; - /* metadata groups contains information for potential retries and + /* metadata group contains information for potential retries and * if group was already fetched successfully */ - struct flb_filter_aws_metadata_group metadata_groups[FLB_FILTER_AWS_METADATA_GROUP_LEN]; + struct flb_filter_aws_metadata_group group_base; + struct flb_filter_aws_metadata_group group_tag; int metadata_retrieved; /* Plugin can use EC2 metadata v1 or v2; default is v2 */ From a398d6031e2cc54191f3d05b416e46754344bee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Warzy=C5=84ski?= Date: Sat, 1 Jul 2023 21:59:35 +0200 Subject: [PATCH 10/13] filter: aws get_ec2_metadata make one bool var MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Warzyński --- plugins/filter_aws/aws.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/filter_aws/aws.c b/plugins/filter_aws/aws.c index eccb1c20444..1045fc8c0bb 100644 --- a/plugins/filter_aws/aws.c +++ b/plugins/filter_aws/aws.c @@ -752,8 +752,7 @@ static int ec2_metadata_group_should_fetch(struct flb_filter_aws_metadata_group static int get_ec2_metadata(struct flb_filter_aws *ctx) { int ret; - int no_failures = FLB_TRUE; - int no_fetches_skipped = FLB_TRUE; + int metadata_fetched = FLB_TRUE; if (!ctx->group_base.done) { ret = get_ec2_metadata_base(ctx); @@ -765,7 +764,7 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx) if (!ctx->group_tag.done) { if (!ec2_metadata_group_should_fetch(&ctx->group_tag)) { - no_fetches_skipped = FLB_FALSE; + metadata_fetched = FLB_FALSE; } else { ret = get_ec2_metadata_tags(ctx); if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { @@ -774,12 +773,12 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx) if (ret == 0) { ctx->group_tag.done = FLB_TRUE; } else { - no_failures = FLB_FALSE; + metadata_fetched = FLB_FALSE; } } } - if (no_failures && no_fetches_skipped) { + if (metadata_fetched) { ctx->metadata_retrieved = FLB_TRUE; } From f48020f9a457309ff2d2f109bf7de1da801857b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Warzy=C5=84ski?= Date: Sat, 1 Jul 2023 23:27:43 +0200 Subject: [PATCH 11/13] test: filter aws tests for base metadata groups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add test cases for injecting following information: - instance_id - instance_type - private_ip - vpc_id - ami_id - account_id - hostname - az Signed-off-by: Mateusz Warzyński --- tests/runtime/filter_aws.c | 803 +++++++++++++++++++++++++++++++++++++ 1 file changed, 803 insertions(+) diff --git a/tests/runtime/filter_aws.c b/tests/runtime/filter_aws.c index 2d682d3256d..f25430b1bda 100644 --- a/tests/runtime/filter_aws.c +++ b/tests/runtime/filter_aws.c @@ -42,6 +42,801 @@ int callback_test(void* data, size_t size, void* cb_data) return 0; } +void flb_test_aws_instance_id() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/instance-id/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "i-0e66fc7f9809d7168"), + set(PAYLOAD_SIZE, 19) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"ec2_instance_id\":\"i-0e66fc7f9809d7168\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_instance_type() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/instance-type/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "t2.micro"), + set(PAYLOAD_SIZE, 8) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_type", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"ec2_instance_type\":\"t2.micro\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_private_ip() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/local-ipv4/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "10.158.112.84"), + set(PAYLOAD_SIZE, 13) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "private_ip", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"private_ip\":\"10.158.112.84\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_vpc_id() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/mac/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "00:00:5e:00:53:af"), + set(PAYLOAD_SIZE, 17) + ), + response( + expect(URI, "/latest/meta-data/network/interfaces/macs/00:00:5e:00:53:af/vpc-id/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "vpc-2928ea42"), + set(PAYLOAD_SIZE, 12) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "vpc_id", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"vpc_id\":\"vpc-2928ea42\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_ami_id() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/ami-id/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "ami-5fb8c835"), + set(PAYLOAD_SIZE, 12) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ami_id", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"ami_id\":\"ami-5fb8c835\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_account_id() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/dynamic/instance-identity/document/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "{\"devpayProductCodes\":null,\"marketplaceProductCodes\":[\"1abc2defghijklm3nopqrs4tu\"],\"availabilityZone\":\"us-east-1a\",\"privateIp\":\"10.158.112.84\",\"version\":\"2017-09-30\",\"instanceId\":\"i-1234567890abcdef0\",\"billingProducts\":null,\"instanceType\":\"t2.micro\",\"accountId\":\"123456789012\",\"imageId\":\"ami-5fb8c835\",\"pendingTime\":\"2016-11-19T16:32:11Z\",\"architecture\":\"x86_64\",\"kernelId\":null,\"ramdiskId\":null,\"region\":\"us-east-1\"}"), + set(PAYLOAD_SIZE, 417) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "account_id", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"account_id\":\"123456789012\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_hostname() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/hostname/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "ip-10-158-112-84.us-west-2.compute.internal"), + set(PAYLOAD_SIZE, 43) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "hostname", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"hostname\":\"ip-10-158-112-84.us-west-2.compute.internal\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_az() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/placement/availability-zone/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "us-east-1a"), + set(PAYLOAD_SIZE, 10) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "true", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"az\":\"us-east-1a\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + void flb_test_aws_ec2_tags_present() { int ret; int bytes; @@ -848,6 +1643,14 @@ void flb_test_aws_ec2_tags_exclude() { TEST_LIST = { + {"aws_instance_id", flb_test_aws_instance_id}, + {"aws_instance_type", flb_test_aws_instance_type}, + {"aws_private_ip", flb_test_aws_private_ip}, + {"aws_vpc_id", flb_test_aws_vpc_id}, + {"aws_ami_id", flb_test_aws_ami_id}, + {"aws_account_id", flb_test_aws_account_id}, + {"aws_hostname", flb_test_aws_hostname}, + {"aws_az", flb_test_aws_az}, {"aws_ec2_tags_present", flb_test_aws_ec2_tags_present}, {"aws_ec2_tags_404", flb_test_aws_ec2_tags_404}, {"aws_ec2_tags_list_500", flb_test_aws_ec2_tags_list_500}, From 73bd05cfe1dc4feb92461deead8b72a0b36f6381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Warzy=C5=84ski?= Date: Sat, 1 Jul 2023 22:51:30 +0200 Subject: [PATCH 12/13] filter: aws retry all metadata groups in interval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 'retry_interval_s' is specified in the filter aws configuration Signed-off-by: Mateusz Warzyński --- plugins/filter_aws/aws.c | 255 +++++++++++++++++++++++++++++++-------- plugins/filter_aws/aws.h | 17 ++- 2 files changed, 214 insertions(+), 58 deletions(-) diff --git a/plugins/filter_aws/aws.c b/plugins/filter_aws/aws.c index 1045fc8c0bb..acb5b7d7af2 100644 --- a/plugins/filter_aws/aws.c +++ b/plugins/filter_aws/aws.c @@ -54,57 +54,84 @@ static void expose_aws_meta(struct flb_filter_aws *ctx) flb_env_set(env, "aws", "enabled"); - if (ctx->group_base.done && - !ctx->group_base.exposed) { + if (ctx->group_az.done && + !ctx->group_az.exposed) { if (ctx->availability_zone_include) { flb_env_set(env, "aws." FLB_FILTER_AWS_AVAILABILITY_ZONE_KEY, ctx->availability_zone); } + ctx->group_az.exposed = FLB_TRUE; + } + if (ctx->group_instance_id.done && + !ctx->group_instance_id.exposed) { if (ctx->instance_id_include) { flb_env_set(env, "aws." FLB_FILTER_AWS_INSTANCE_ID_KEY, ctx->instance_id); } + ctx->group_instance_id.exposed = FLB_TRUE; + } + if (ctx->group_instance_type.done && + !ctx->group_instance_type.exposed) { if (ctx->instance_type_include) { flb_env_set(env, "aws." FLB_FILTER_AWS_INSTANCE_TYPE_KEY, ctx->instance_type); } + ctx->group_instance_type.exposed = FLB_TRUE; + } + if (ctx->group_private_ip.done && + !ctx->group_private_ip.exposed) { if (ctx->private_ip_include) { flb_env_set(env, "aws." FLB_FILTER_AWS_PRIVATE_IP_KEY, ctx->private_ip); } + ctx->group_private_ip.exposed = FLB_TRUE; + } + if (ctx->group_vpc_id.done && + !ctx->group_vpc_id.exposed) { if (ctx->vpc_id_include) { flb_env_set(env, "aws." FLB_FILTER_AWS_VPC_ID_KEY, ctx->vpc_id); } + ctx->group_vpc_id.exposed = FLB_TRUE; + } + if (ctx->group_ami_id.done && + !ctx->group_ami_id.exposed) { if (ctx->ami_id_include) { flb_env_set(env, "aws." FLB_FILTER_AWS_AMI_ID_KEY, ctx->ami_id); } + ctx->group_ami_id.exposed = FLB_TRUE; + } + if (ctx->group_account_id.done && + !ctx->group_account_id.exposed) { if (ctx->account_id_include) { flb_env_set(env, "aws." FLB_FILTER_AWS_ACCOUNT_ID_KEY, ctx->account_id); } + ctx->group_account_id.exposed = FLB_TRUE; + } + if (ctx->group_hostname.done && + !ctx->group_hostname.exposed) { if (ctx->hostname_include) { flb_env_set(env, "aws." FLB_FILTER_AWS_HOSTNAME_KEY, ctx->hostname); } - - ctx->group_base.exposed = FLB_TRUE; + ctx->group_hostname.exposed = FLB_TRUE; } /* TODO: expose aws ec2 tags in flb_env_set */ @@ -609,18 +636,10 @@ static int get_ec2_tags(struct flb_filter_aws *ctx) return 0; } -/* - * Makes a call to IMDS to set get the values of all metadata fields. - * It can be called repeatedly if some metadata calls initially do not succeed. - * However, if function succeeds, the expectation is that it shouldn't be called again. - */ -static int get_ec2_metadata_base(struct flb_filter_aws *ctx) +static int get_ec2_metadata_instance_id(struct flb_filter_aws *ctx) { int ret; - ctx->group_base. - last_fetch_attempt = time(NULL); - if (ctx->instance_id_include && !ctx->instance_id) { ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_INSTANCE_ID_PATH, &ctx->instance_id, @@ -631,16 +650,12 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) } } - if (ctx->availability_zone_include && !ctx->availability_zone) { - ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_AZ_PATH, - &ctx->availability_zone, - &ctx->availability_zone_len); + return 0; +} - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to get instance AZ"); - return -1; - } - } +static int get_ec2_metadata_instance_type(struct flb_filter_aws *ctx) +{ + int ret; if (ctx->instance_type_include && !ctx->instance_type) { ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_INSTANCE_TYPE_PATH, @@ -652,6 +667,13 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) } } + return 0; +} + +static int get_ec2_metadata_private_ip(struct flb_filter_aws *ctx) +{ + int ret; + if (ctx->private_ip_include && !ctx->private_ip) { ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_PRIVATE_IP_PATH, &ctx->private_ip, &ctx->private_ip_len); @@ -662,6 +684,13 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) } } + return 0; +} + +static int get_ec2_metadata_vpc_id(struct flb_filter_aws *ctx) +{ + int ret; + if (ctx->vpc_id_include && !ctx->vpc_id) { ret = get_vpc_id(ctx); @@ -671,6 +700,13 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) } } + return 0; +} + +static int get_ec2_metadata_ami_id(struct flb_filter_aws *ctx) +{ + int ret; + if (ctx->ami_id_include && !ctx->ami_id) { ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_AMI_ID_PATH, &ctx->ami_id, &ctx->ami_id_len); @@ -681,6 +717,13 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) } } + return 0; +} + +static int get_ec2_metadata_account_id(struct flb_filter_aws *ctx) +{ + int ret; + if (ctx->account_id_include && !ctx->account_id) { ret = flb_aws_imds_request_by_key(ctx->client_imds, FLB_AWS_IMDS_ACCOUNT_ID_PATH, &ctx->account_id, &ctx->account_id_len, @@ -692,6 +735,14 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) } } + + return 0; +} + +static int get_ec2_metadata_hostname(struct flb_filter_aws *ctx) +{ + int ret; + if (ctx->hostname_include && !ctx->hostname) { ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_HOSTNAME_PATH, &ctx->hostname, &ctx->hostname_len); @@ -705,12 +756,28 @@ static int get_ec2_metadata_base(struct flb_filter_aws *ctx) return 0; } -static int get_ec2_metadata_tags(struct flb_filter_aws *ctx) +static int get_ec2_metadata_az(struct flb_filter_aws *ctx) { int ret; - ctx->group_tag. - last_fetch_attempt = time(NULL); + if (ctx->availability_zone_include && !ctx->availability_zone) { + ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_AZ_PATH, + &ctx->availability_zone, + &ctx->availability_zone_len); + + if (ret < 0) { + flb_plg_error(ctx->ins, "Failed to get instance AZ"); + return -1; + } + } + + return 0; +} + + +static int get_ec2_metadata_tags(struct flb_filter_aws *ctx) +{ + int ret; if (ctx->tags_enabled && !ctx->tags_fetched) { ret = get_ec2_tags(ctx); @@ -723,11 +790,12 @@ static int get_ec2_metadata_tags(struct flb_filter_aws *ctx) return 0; } -static int ec2_metadata_group_should_fetch(struct flb_filter_aws_metadata_group *group) +static int ec2_metadata_group_should_fetch(struct flb_filter_aws *ctx, + struct flb_filter_aws_metadata_group *group) { time_t now, required_interval, interval; - required_interval = group->retry_required_interval; + required_interval = ctx->retry_required_interval; if (required_interval == 0) { return FLB_TRUE; } @@ -742,6 +810,25 @@ static int ec2_metadata_group_should_fetch(struct flb_filter_aws_metadata_group return FLB_TRUE; } +static int get_ec2_metadata_group(struct flb_filter_aws *ctx, + struct flb_filter_aws_metadata_group *group, + int (*fetch_func)(struct flb_filter_aws *ctx)) +{ + int ret; + if (group->done) { + return 0; + } + if (!ec2_metadata_group_should_fetch(ctx, group)) { + return -1; + } + group->last_fetch_attempt = time(NULL); + ret = fetch_func(ctx); + if (ret == 0) { + group->done = FLB_TRUE; + } + return ret; +} + /* * Fetches all metadata values, including tags, from IMDS. * Function handles retries as configured for each metadata group. @@ -754,28 +841,84 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx) int ret; int metadata_fetched = FLB_TRUE; - if (!ctx->group_base.done) { - ret = get_ec2_metadata_base(ctx); - if (ret < 0) { + if (ctx->metadata_retrieved) { + return 0; + } + + ret = get_ec2_metadata_group(ctx, &ctx->group_instance_id, + get_ec2_metadata_instance_id); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { return ret; } - ctx->group_base.done = FLB_TRUE; + metadata_fetched = FLB_FALSE; } - if (!ctx->group_tag.done) { - if (!ec2_metadata_group_should_fetch(&ctx->group_tag)) { - metadata_fetched = FLB_FALSE; - } else { - ret = get_ec2_metadata_tags(ctx); - if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { - return ret; - } - if (ret == 0) { - ctx->group_tag.done = FLB_TRUE; - } else { - metadata_fetched = FLB_FALSE; - } + ret = get_ec2_metadata_group(ctx, &ctx->group_account_id, + get_ec2_metadata_account_id); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + metadata_fetched = FLB_FALSE; + } + + ret = get_ec2_metadata_group(ctx, &ctx->group_instance_type, + get_ec2_metadata_instance_type); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + metadata_fetched = FLB_FALSE; + } + + ret = get_ec2_metadata_group(ctx, &ctx->group_private_ip, + get_ec2_metadata_private_ip); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + metadata_fetched = FLB_FALSE; + } + + ret = get_ec2_metadata_group(ctx, &ctx->group_vpc_id, get_ec2_metadata_vpc_id); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; } + metadata_fetched = FLB_FALSE; + } + + ret = get_ec2_metadata_group(ctx, &ctx->group_ami_id, get_ec2_metadata_ami_id); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + metadata_fetched = FLB_FALSE; + } + + ret = get_ec2_metadata_group(ctx, &ctx->group_hostname, get_ec2_metadata_hostname); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + metadata_fetched = FLB_FALSE; + } + + ret = get_ec2_metadata_group(ctx, &ctx->group_az, get_ec2_metadata_az); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + metadata_fetched = FLB_FALSE; + } + + ret = get_ec2_metadata_group(ctx, &ctx->group_tag, get_ec2_metadata_tags); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + metadata_fetched = FLB_FALSE; } if (metadata_fetched) { @@ -808,10 +951,11 @@ static int cb_aws_filter(const void *data, size_t bytes, /* First check that the metadata has been retrieved */ if (!ctx->metadata_retrieved) { - ret = get_ec2_metadata(ctx); - if (ret < 0) { - return FLB_FILTER_NOTOUCH; - } + get_ec2_metadata(ctx); /* ignore the error */ + /* it assumes the error cannot be the configuration error, as it would fail */ + /* during the _init function; */ + /* even if get_ec2_metadata failed, it may have been able to fetch some groups */ + /* therefore we continue flushing the logs with what we've got */ expose_aws_meta(ctx); } @@ -864,6 +1008,7 @@ static int cb_aws_filter(const void *data, size_t bytes, /* append new keys */ if (ctx->availability_zone_include && + ctx->group_az.done && ret == FLB_EVENT_ENCODER_SUCCESS) { ret = flb_log_event_encoder_append_body_values( &log_encoder, @@ -873,6 +1018,7 @@ static int cb_aws_filter(const void *data, size_t bytes, } if (ctx->instance_id_include && + ctx->group_instance_id.done && ret == FLB_EVENT_ENCODER_SUCCESS) { ret = flb_log_event_encoder_append_body_values( &log_encoder, @@ -882,6 +1028,7 @@ static int cb_aws_filter(const void *data, size_t bytes, } if (ctx->instance_type_include && + ctx->group_instance_type.done && ret == FLB_EVENT_ENCODER_SUCCESS) { ret = flb_log_event_encoder_append_body_values( &log_encoder, @@ -891,6 +1038,7 @@ static int cb_aws_filter(const void *data, size_t bytes, } if (ctx->private_ip_include && + ctx->group_private_ip.done && ret == FLB_EVENT_ENCODER_SUCCESS) { ret = flb_log_event_encoder_append_body_values( &log_encoder, @@ -900,6 +1048,7 @@ static int cb_aws_filter(const void *data, size_t bytes, } if (ctx->vpc_id_include && + ctx->group_vpc_id.done && ret == FLB_EVENT_ENCODER_SUCCESS) { ret = flb_log_event_encoder_append_body_values( &log_encoder, @@ -909,6 +1058,7 @@ static int cb_aws_filter(const void *data, size_t bytes, } if (ctx->ami_id_include && + ctx->group_ami_id.done && ret == FLB_EVENT_ENCODER_SUCCESS) { ret = flb_log_event_encoder_append_body_values( &log_encoder, @@ -918,6 +1068,7 @@ static int cb_aws_filter(const void *data, size_t bytes, } if (ctx->account_id_include && + ctx->group_account_id.done && ret == FLB_EVENT_ENCODER_SUCCESS) { ret = flb_log_event_encoder_append_body_values( &log_encoder, @@ -927,6 +1078,7 @@ static int cb_aws_filter(const void *data, size_t bytes, } if (ctx->hostname_include && + ctx->group_hostname.done && ret == FLB_EVENT_ENCODER_SUCCESS) { ret = flb_log_event_encoder_append_body_values( &log_encoder, @@ -1117,11 +1269,8 @@ static struct flb_config_map config_map[] = { }, { FLB_CONFIG_MAP_INT, "retry_interval_s", "300", - /* for the time being, the only group which has retries is ec2 tags */ - /* therefore configuration immediately sets this value just for ec2 tags */ - 0, FLB_TRUE, offsetof(struct flb_filter_aws, - group_tag.retry_required_interval), - "Defines minimum duration between retries for fetching EC2 instance tags" + 0, FLB_TRUE, offsetof(struct flb_filter_aws, retry_required_interval), + "Defines minimum duration between retries for fetching metadata groups" }, {0} }; diff --git a/plugins/filter_aws/aws.h b/plugins/filter_aws/aws.h index c6458388f73..f369f0bbcc7 100644 --- a/plugins/filter_aws/aws.h +++ b/plugins/filter_aws/aws.h @@ -56,12 +56,8 @@ struct flb_filter_aws_metadata_group { /* defines a timestamp of last execution of fetch method related to the group */ /* unit: timestamp in seconds */ time_t last_fetch_attempt; - /* defines a minimal interval before consecutive retries */ - /* unit: seconds */ - time_t retry_required_interval; }; - struct flb_filter_aws { struct flb_filter_aws_init_options *options; @@ -136,8 +132,19 @@ struct flb_filter_aws { /* metadata group contains information for potential retries and * if group was already fetched successfully */ - struct flb_filter_aws_metadata_group group_base; + struct flb_filter_aws_metadata_group group_az; + struct flb_filter_aws_metadata_group group_instance_id; + struct flb_filter_aws_metadata_group group_instance_type; + struct flb_filter_aws_metadata_group group_private_ip; + struct flb_filter_aws_metadata_group group_vpc_id; + struct flb_filter_aws_metadata_group group_ami_id; + struct flb_filter_aws_metadata_group group_account_id; + struct flb_filter_aws_metadata_group group_hostname; struct flb_filter_aws_metadata_group group_tag; + /* defines a minimal interval before consecutive retries */ + /* unit: seconds */ + time_t retry_required_interval; + /* defines if all metadata groups were fetched successfully */ int metadata_retrieved; /* Plugin can use EC2 metadata v1 or v2; default is v2 */ From 9216df6d92669686756f513a05291d1a29108f7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Warzy=C5=84ski?= Date: Tue, 26 Mar 2024 02:06:43 +0100 Subject: [PATCH 13/13] filter: aws use '== FLB_TRUE' for if bool conditions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Warzyński --- plugins/filter_aws/aws.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/filter_aws/aws.c b/plugins/filter_aws/aws.c index acb5b7d7af2..afe94a8a466 100644 --- a/plugins/filter_aws/aws.c +++ b/plugins/filter_aws/aws.c @@ -921,7 +921,7 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx) metadata_fetched = FLB_FALSE; } - if (metadata_fetched) { + if (metadata_fetched == FLB_TRUE) { ctx->metadata_retrieved = FLB_TRUE; }