From 6af61749af8f5de90678b582a51e545c7e40ab8a Mon Sep 17 00:00:00 2001 From: braydonk Date: Sat, 28 Oct 2023 15:49:37 +0000 Subject: [PATCH] flb_parser: add time_system_timezone Added a new config option for parsers called `time_system_timezone` which will fall back to `mktime` for tm to time conversion which will force the usage of the system timezone. --- include/fluent-bit/flb_parser.h | 12 +- .../in_kubernetes_events/kubernetes_events.c | 2 +- src/flb_parser.c | 25 +- src/flb_parser_json.c | 2 +- src/flb_parser_logfmt.c | 2 +- src/flb_parser_ltsv.c | 2 +- src/flb_parser_regex.c | 2 +- src/multiline/flb_ml_parser_cri.c | 1 + src/multiline/flb_ml_parser_docker.c | 1 + tests/internal/parser.c | 182 ++++++++++++++- tests/internal/parser_json.c | 219 +++++++++++++----- tests/internal/parser_logfmt.c | 102 +++++++- tests/internal/parser_ltsv.c | 104 ++++++++- tests/internal/parser_regex.c | 105 ++++++++- tests/runtime/filter_parser.c | 166 ++++++++++++- 15 files changed, 822 insertions(+), 105 deletions(-) diff --git a/include/fluent-bit/flb_parser.h b/include/fluent-bit/flb_parser.h index 7d5cfcbc74f..4206cfbeded 100644 --- a/include/fluent-bit/flb_parser.h +++ b/include/fluent-bit/flb_parser.h @@ -48,6 +48,7 @@ struct flb_parser { char *time_fmt_full; /* original given time format */ char *time_key; /* field name that contains the time */ int time_offset; /* fixed UTC offset */ + int time_system_timezone; /* use the system timezone as a fallback */ int time_keep; /* keep time field */ int time_strict; /* parse time field strictly */ int logfmt_no_bare_keys; /* in logfmt parsers, require all keys to have values */ @@ -74,13 +75,19 @@ enum { FLB_PARSER_TYPE_HEX, }; -static inline time_t flb_parser_tm2time(const struct flb_tm *src) +static inline time_t flb_parser_tm2time(const struct flb_tm *src, + int use_system_timezone) { struct tm tmp; time_t res; tmp = src->tm; - res = timegm(&tmp) - flb_tm_gmtoff(src); + if (use_system_timezone) { + tmp.tm_isdst = -1; + res = mktime(&tmp); + } else { + res = timegm(&tmp) - flb_tm_gmtoff(src); + } return res; } @@ -92,6 +99,7 @@ struct flb_parser *flb_parser_create(const char *name, const char *format, const char *time_offset, int time_keep, int time_strict, + int time_system_timezone, int logfmt_no_bare_keys, struct flb_parser_types *types, int types_len, diff --git a/plugins/in_kubernetes_events/kubernetes_events.c b/plugins/in_kubernetes_events/kubernetes_events.c index 97719fba6e0..39a832f49fa 100644 --- a/plugins/in_kubernetes_events/kubernetes_events.c +++ b/plugins/in_kubernetes_events/kubernetes_events.c @@ -175,7 +175,7 @@ static int timestamp_lookup(struct k8s_events *ctx, char *ts, struct flb_time *t return -1; } - time->tm.tv_sec = flb_parser_tm2time(&tm); + time->tm.tv_sec = flb_parser_tm2time(&tm, FLB_FALSE); time->tm.tv_nsec = 0; return 0; diff --git a/src/flb_parser.c b/src/flb_parser.c index 4ccecc91b1d..8d190ff8156 100644 --- a/src/flb_parser.c +++ b/src/flb_parser.c @@ -152,6 +152,7 @@ struct flb_parser *flb_parser_create(const char *name, const char *format, const char *time_offset, int time_keep, int time_strict, + int time_system_timezone, int logfmt_no_bare_keys, struct flb_parser_types *types, int types_len, @@ -312,8 +313,17 @@ struct flb_parser *flb_parser_create(const char *name, const char *format, p->time_frac_secs = (tmp + 2); } - /* Optional fixed timezone offset */ - if (time_offset) { + /* + * Fall back to the system timezone + * if there is no zone parsed from the log. + */ + p->time_system_timezone = time_system_timezone; + + /* + * Optional fixed timezone offset, only applied if + * not falling back to system timezone. + */ + if (!p->time_system_timezone && time_offset) { diff = 0; len = strlen(time_offset); ret = flb_parser_tzone_offset(time_offset, len, &diff); @@ -487,6 +497,7 @@ static int parser_conf_file(const char *cfg, struct flb_cf *cf, int skip_empty; int time_keep; int time_strict; + int time_system_timezone; int logfmt_no_bare_keys; int types_len; struct mk_list *head; @@ -561,6 +572,13 @@ static int parser_conf_file(const char *cfg, struct flb_cf *cf, flb_sds_destroy(tmp_str); } + time_system_timezone = FLB_FALSE; + tmp_str = get_parser_key(config, cf, s, "time_system_timezone"); + if (tmp_str) { + time_system_timezone = flb_utils_bool(tmp_str); + flb_sds_destroy(tmp_str); + } + /* time_offset (UTC offset) */ time_offset = get_parser_key(config, cf, s, "time_offset"); @@ -587,7 +605,8 @@ static int parser_conf_file(const char *cfg, struct flb_cf *cf, /* Create the parser context */ if (!flb_parser_create(name, format, regex, skip_empty, time_fmt, time_key, time_offset, time_keep, time_strict, - logfmt_no_bare_keys, types, types_len, decoders, config)) { + time_system_timezone, logfmt_no_bare_keys, types, types_len, + decoders, config)) { goto fconf_error; } diff --git a/src/flb_parser_json.c b/src/flb_parser_json.c index 7add06e94fb..34c9bd379c9 100644 --- a/src/flb_parser_json.c +++ b/src/flb_parser_json.c @@ -207,7 +207,7 @@ int flb_parser_json_do(struct flb_parser *parser, skip = map_size; } else { - time_lookup = flb_parser_tm2time(&tm); + time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone); } /* Compose a new map without the time_key field */ diff --git a/src/flb_parser_logfmt.c b/src/flb_parser_logfmt.c index 8e6b465904a..14fcafbb04f 100644 --- a/src/flb_parser_logfmt.c +++ b/src/flb_parser_logfmt.c @@ -166,7 +166,7 @@ static int logfmt_parser(struct flb_parser *parser, parser->name, parser->time_fmt_full); return -1; } - *time_lookup = flb_parser_tm2time(&tm); + *time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone); } time_found = FLB_TRUE; } diff --git a/src/flb_parser_ltsv.c b/src/flb_parser_ltsv.c index 8f38102cf7c..3e4073c2c59 100644 --- a/src/flb_parser_ltsv.c +++ b/src/flb_parser_ltsv.c @@ -139,7 +139,7 @@ static int ltsv_parser(struct flb_parser *parser, parser->name, parser->time_fmt_full); return -1; } - *time_lookup = flb_parser_tm2time(&tm); + *time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone); } time_found = FLB_TRUE; } diff --git a/src/flb_parser_regex.c b/src/flb_parser_regex.c index efcc6fb60b4..21659cdf473 100644 --- a/src/flb_parser_regex.c +++ b/src/flb_parser_regex.c @@ -87,7 +87,7 @@ static void cb_results(const char *name, const char *value, } pcb->time_frac = frac; - pcb->time_lookup = flb_parser_tm2time(&tm); + pcb->time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone); if (parser->time_keep == FLB_FALSE) { pcb->num_skipped++; diff --git a/src/multiline/flb_ml_parser_cri.c b/src/multiline/flb_ml_parser_cri.c index 669fa39a249..ce32d159d3a 100644 --- a/src/multiline/flb_ml_parser_cri.c +++ b/src/multiline/flb_ml_parser_cri.c @@ -40,6 +40,7 @@ static struct flb_parser *cri_parser_create(struct flb_config *config) NULL, /* time offset */ FLB_TRUE, /* time keep */ FLB_FALSE, /* time strict */ + FLB_FALSE, /* time system timezone */ FLB_FALSE, /* no bare keys */ NULL, /* parser types */ 0, /* types len */ diff --git a/src/multiline/flb_ml_parser_docker.c b/src/multiline/flb_ml_parser_docker.c index 5b622d32c62..23f966ce7a1 100644 --- a/src/multiline/flb_ml_parser_docker.c +++ b/src/multiline/flb_ml_parser_docker.c @@ -35,6 +35,7 @@ static struct flb_parser *docker_parser_create(struct flb_config *config) NULL, /* time offset */ FLB_TRUE, /* time keep */ FLB_FALSE, /* time strict */ + FLB_FALSE, /* time system timezone */ FLB_FALSE, /* no bare keys */ NULL, /* parser types */ 0, /* types len */ diff --git a/tests/internal/parser.c b/tests/internal/parser.c index 6a560db716c..52dc5f44314 100644 --- a/tests/internal/parser.c +++ b/tests/internal/parser.c @@ -124,7 +124,7 @@ int flb_parser_regex_do(struct flb_parser *parser, struct flb_time *out_time); /* Parse timezone string and get the offset */ -void test_parser_tzone_offset() +void test_parser_tzone_offset(void) { int i; int len; @@ -167,7 +167,7 @@ static void load_regex_parsers(struct flb_config *config) TEST_CHECK(ret == 0); } -void test_parser_time_lookup() +void test_parser_time_lookup(void) { int i; int j; @@ -231,7 +231,7 @@ void test_parser_time_lookup() continue; } - epoch = flb_parser_tm2time(&tm); + epoch = flb_parser_tm2time(&tm, FLB_FALSE); epoch -= year_diff; TEST_CHECK(t->epoch == epoch); TEST_CHECK(t->frac_seconds == ns); @@ -246,7 +246,7 @@ void test_parser_time_lookup() } /* Do time lookup using the JSON parser backend*/ -void test_json_parser_time_lookup() +void test_json_parser_time_lookup(void) { int i; int j; @@ -331,7 +331,7 @@ void test_json_parser_time_lookup() } /* Do time lookup using the Regex parser backend*/ -void test_regex_parser_time_lookup() +void test_regex_parser_time_lookup(void) { int i; int j; @@ -487,8 +487,7 @@ static int a_mysql_unquote_test(struct flb_parser *p, char *source, char *expect return 1; } - -void test_mysql_unquoted() +void test_mysql_unquoted(void) { struct flb_parser *p; struct flb_config *config; @@ -513,16 +512,183 @@ void test_mysql_unquoted() flb_parser_exit(config); flb_config_exit(config); +} + +/** + * I've opted to leave this function around for future testers. + * When working with these C time structures, it can really help + * to just sanity check what the timestamp actually is under the + * hood. If you are adding or working with the timestamp tests, + * you can manually add this function to print out a human readable + * timestamp. Note that it will be in your system's local timezone; + * you'll have to do some math based on the test you're working with. + * - Braydon Kains, @braydonk + */ +void debug_print_local_timestamp(time_t t) +{ + struct tm tm; + char buf[80]; + tm = *localtime(&t); + strftime(buf, sizeof(buf), "%a %Y-%m-%d %H:%M:%S %Z", &tm); + printf("%s\n", buf); +} + +/** + * Run tm2time on a flb_tm and expect a certain epoch (in seconds). + */ +void tm2time_testcase(struct flb_tm *tm, time_t expected_epoch, int system_timezone) +{ + time_t result_epoch = flb_parser_tm2time(tm, system_timezone); + TEST_CHECK_(result_epoch == expected_epoch, + "expected (%lld), got (%lld)", + (long long)expected_epoch, (long long)result_epoch); +} + +/** + * Perform a tm2time_testcase, but first set the specified timezone, and + * reset the timezone after. + */ +void set_tz_tm2time_testcase(char *tz, struct flb_tm *tm, time_t expected_epoch) +{ + int ret; + char *original_tz = NULL; + + /* Set the timezone and keep the original value to reset it later. */ + if (tz) { + original_tz = getenv("TZ"); + ret = setenv("TZ", tz, 1); + TEST_ASSERT_(ret == 0, + "failed to set timezone to %s", tz); + } + + tm2time_testcase(tm, expected_epoch, FLB_TRUE); + + /* Reset the timezone. */ + if (original_tz) { + ret = setenv("TZ", original_tz, 1); + TEST_ASSERT_(ret == 0, + "failed to restore timezone to %s", original_tz); + flb_free(original_tz); + } else { + ret = unsetenv("TZ"); + TEST_ASSERT_(ret == 0, + "failed to restore original timezone setting"); + } +} + +void test_tm2time_timegm(void) +{ + /** + * TZ=GMT date -d @1673654400 + * Mon Feb 20 12:00:00 AM EST 2023 + */ + time_t expected_epoch = 1673654400; + struct flb_tm *tm = malloc(sizeof(struct tm)); + tm->tm.tm_year = 2023-1900; + tm->tm.tm_mon = 0; + tm->tm.tm_mday = 14; + tm->tm.tm_hour = 0; + tm->tm.tm_min = 0; + tm->tm.tm_sec = 0; + tm2time_testcase(tm, expected_epoch, FLB_FALSE); + + flb_free(tm); +} + +void test_tm2time_timegm_with_gmtoff(void) +{ + /** + * TZ=GMT date -d @1676869200 + * Mon Feb 20 05:00:00 AM GMT 2023 + * + * TZ=EST date -d @1676869200 + * Mon Feb 20 12:00:00 AM EST 2023 + */ + time_t expected_epoch = 1676869200; + struct flb_tm *tm = malloc(sizeof(struct flb_tm)); + + tm->tm.tm_year = 2023-1900; + tm->tm.tm_mon = 1; + tm->tm.tm_mday = 20; + tm->tm.tm_hour = 0; + tm->tm.tm_min = 0; + tm->tm.tm_sec = 0; + /** + * tm_gmtoff is the number of seconds east of GMT. + * This calculation equates to UTC-5, which in January + * is EST. + */ + flb_tm_gmtoff(tm) = (-5)*3600; + + tm2time_testcase(tm, expected_epoch, FLB_FALSE); + + flb_free(tm); +} + +void test_tm2time_system_timezone(void) +{ + /** + * TZ=MST7MDT date -d @1676876400 + * Mon Feb 20 12:00:00 AM MST 2023 + * + * TZ=GMT date -d @1676876400 + * Mon Feb 20 07:00:00 AM GMT 2023 + */ + time_t expected_epoch = 1676876400; + struct flb_tm *tm = malloc(sizeof(struct flb_tm)); + + tm->tm.tm_year = 2023-1900; + tm->tm.tm_mon = 1; + tm->tm.tm_mday = 20; + tm->tm.tm_hour = 0; + tm->tm.tm_min = 0; + tm->tm.tm_sec = 0; + + set_tz_tm2time_testcase("MST7MDT", tm, expected_epoch); + + flb_free(tm); } +void test_tm2time_mktime_system_timezone_find_dst(void) +{ + /** + * TZ=GMT date -d @1587355200 + * Mon Apr 20 04:00:00 AM GMT 2020 + * + * TZ=EST5EDT date -d @1587355200 + * Mon Apr 20 12:00:00 AM EDT 2020 + */ + time_t expected_epoch = 1587355200; + struct flb_tm *tm = malloc(sizeof(struct flb_tm)); + + tm->tm.tm_year = 2020-1900; + tm->tm.tm_mon = 3; + tm->tm.tm_mday = 20; + tm->tm.tm_hour = 0; + tm->tm.tm_min = 0; + tm->tm.tm_sec = 0; + + /** + * In this test case, tm2time should be able automatically + * calculating and determining EDT is correct based on the + * date of April 20, which is after the daylight savings switch. + */ + set_tz_tm2time_testcase("EST5EDT", tm, expected_epoch); + + flb_free(tm); +} TEST_LIST = { { "tzone_offset", test_parser_tzone_offset}, { "time_lookup", test_parser_time_lookup}, { "json_time_lookup", test_json_parser_time_lookup}, { "regex_time_lookup", test_regex_parser_time_lookup}, - { "mysql_unquoted" , test_mysql_unquoted }, + { "mysql_unquoted", test_mysql_unquoted }, + { "tm2time_timegm", test_tm2time_timegm }, + { "tm2time_timegm_with_gmtoff", test_tm2time_timegm_with_gmtoff }, + { "tm2time_system_timezone", test_tm2time_system_timezone }, + { "tm2time_system_timezone_find_dst", test_tm2time_mktime_system_timezone_find_dst }, { 0 } }; diff --git a/tests/internal/parser_json.c b/tests/internal/parser_json.c index 5a9b6eadbfe..27f67d871ea 100644 --- a/tests/internal/parser_json.c +++ b/tests/internal/parser_json.c @@ -171,7 +171,7 @@ void test_basic() } parser = flb_parser_create("json", "json", NULL, FLB_FALSE, NULL, NULL, NULL, - FLB_FALSE, FLB_FALSE, FLB_FALSE, + FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE, NULL, 0, NULL, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -223,7 +223,7 @@ void test_time_key() } parser = flb_parser_create("json", "json", NULL, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL, - FLB_FALSE, FLB_FALSE, FLB_FALSE, + FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE, NULL, 0, NULL, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -284,7 +284,7 @@ void test_time_keep() } parser = flb_parser_create("json", "json", NULL, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL, - FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE, + FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE, FLB_FALSE, NULL, 0, NULL, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -319,6 +319,155 @@ void test_time_keep() flb_config_exit(config); } +void test_time_key_kept_if_parse_fails() +{ + struct flb_parser *parser = NULL; + struct flb_config *config = NULL; + int ret = 0; + char *input = "{\"str\":\"text\", \"time\":\"nonsense\"}"; + char *time_format = "%Y-%m-%dT%H:%M:%S.%L"; + void *out_buf = NULL; + size_t out_size = 0; + struct flb_time out_time; + char *expected_strs[] = {"str", "text", "time", "nonsense"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + out_time.tm.tv_sec = 0; + out_time.tm.tv_nsec = 0; + + config = flb_config_init(); + if(!TEST_CHECK(config != NULL)) { + TEST_MSG("flb_config_init failed"); + exit(1); + } + + parser = flb_parser_create("json", "json", NULL, FLB_FALSE, time_format, "time", NULL, + FLB_FALSE, FLB_TRUE, FLB_FALSE, FLB_FALSE, + NULL, 0, NULL, config); + if (!TEST_CHECK(parser != NULL)) { + TEST_MSG("flb_parser_create failed"); + flb_config_exit(config); + exit(1); + } + + ret = flb_parser_do(parser, input, strlen(input), &out_buf, &out_size, &out_time); + if (!TEST_CHECK(ret != -1)) { + TEST_MSG("flb_parser_do failed"); + flb_parser_destroy(parser); + flb_config_exit(config); + exit(1); + } + + ret = compare_msgpack(out_buf, out_size, &expected); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("compare failed"); + flb_free(out_buf); + flb_parser_destroy(parser); + flb_config_exit(config); + exit(1); + } + + flb_free(out_buf); + flb_parser_destroy(parser); + flb_config_exit(config); +} + +void test_time_use_system_timezone() +{ + struct flb_parser *parser = NULL; + struct flb_config *config = NULL; + int ret = 0; + char tz[10] = "CET+1CEST\0"; + char *original_tz; + /** + * TZ=CEST date -d @1597104000 + * Tue Aug 11 12:00:00 AM CEST 2020 + * + * TZ=GMT date -d @1597104000 + * Tue Aug 11 12:00:00 AM GMT 2020 + */ + time_t expected_time = 1597104000; + char *input = "{\"str\":\"text\", \"time\":\"2020-08-11T00:00:00\"}"; + char *time_format = "%Y-%m-%dT%H:%M:%S"; + void *out_buf = NULL; + size_t out_size = 0; + struct flb_time out_time; + char *expected_strs[] = {"str", "text"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + out_time.tm.tv_sec = 0; + out_time.tm.tv_nsec = 0; + + config = flb_config_init(); + if(!TEST_CHECK(config != NULL)) { + TEST_MSG("flb_config_init failed"); + exit(1); + } + + parser = flb_parser_create("json", "json", NULL, FLB_FALSE, time_format, "time", NULL, + FLB_FALSE, + FLB_TRUE, /* time strict */ + FLB_TRUE, /* time_system_timezone */ + FLB_FALSE, NULL, 0, NULL, config); + if (!TEST_CHECK(parser != NULL)) { + TEST_MSG("flb_parser_create failed"); + flb_config_exit(config); + exit(1); + } + + /** + * Set the timezone to ensure mktime picks it up. + */ + original_tz = getenv("TZ"); + ret = setenv("TZ", tz, 1); + TEST_ASSERT_(ret == 0, "failed to set timezone to %s", tz); + + ret = flb_parser_do(parser, input, strlen(input), &out_buf, &out_size, &out_time); + if (!TEST_CHECK(ret != -1)) { + TEST_MSG("flb_parser_do failed"); + flb_parser_destroy(parser); + flb_config_exit(config); + exit(1); + } + + ret = compare_msgpack(out_buf, out_size, &expected); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("compare failed"); + flb_free(out_buf); + flb_parser_destroy(parser); + flb_config_exit(config); + exit(1); + } + + if (!TEST_CHECK(out_time.tm.tv_sec == expected_time)) { + TEST_MSG("timestamp error. sec Got=%ld Expect=%"PRIu32"", + out_time.tm.tv_sec, (uint32_t)expected_time); + } + + /** + * Restore original timezone. + */ + if (original_tz) { + ret = setenv("TZ", original_tz, 1); + TEST_ASSERT_(ret == 0, + "failed to restore timezone to %s", original_tz); + flb_free(original_tz); + } else { + ret = unsetenv("TZ"); + } + + flb_free(out_buf); + flb_parser_destroy(parser); + flb_config_exit(config); +} + + /* * JSON parser doesn't support 'types' option. * This test is to check that 'types' doesn't affect output. @@ -365,7 +514,7 @@ void test_types_is_not_supported() types->type = FLB_PARSER_TYPE_HEX; parser = flb_parser_create("json", "json", NULL, FLB_FALSE, NULL, NULL, NULL, - FLB_FALSE, FLB_FALSE, FLB_FALSE, + FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE, types, 1, NULL, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -450,7 +599,7 @@ void test_decode_field_json() } parser = flb_parser_create("json", "json", NULL, FLB_FALSE, NULL, NULL, NULL, - FLB_FALSE, FLB_FALSE, FLB_FALSE, + FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE, NULL, 0, decoder, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -479,69 +628,13 @@ void test_decode_field_json() flb_config_exit(config); } -void test_time_key_kept_if_parse_fails() -{ - struct flb_parser *parser = NULL; - struct flb_config *config = NULL; - int ret = 0; - char *input = "{\"str\":\"text\", \"time\":\"nonsense\"}"; - char *time_format = "%Y-%m-%dT%H:%M:%S.%L"; - void *out_buf = NULL; - size_t out_size = 0; - struct flb_time out_time; - char *expected_strs[] = {"str", "text", "time", "nonsense"}; - struct str_list expected = { - .size = sizeof(expected_strs)/sizeof(char*), - .lists = &expected_strs[0], - }; - - out_time.tm.tv_sec = 0; - out_time.tm.tv_nsec = 0; - - config = flb_config_init(); - if(!TEST_CHECK(config != NULL)) { - TEST_MSG("flb_config_init failed"); - exit(1); - } - - parser = flb_parser_create("json", "json", NULL, FLB_FALSE, time_format, "time", NULL, - FLB_FALSE, FLB_TRUE, FLB_FALSE, - NULL, 0, NULL, config); - if (!TEST_CHECK(parser != NULL)) { - TEST_MSG("flb_parser_create failed"); - flb_config_exit(config); - exit(1); - } - - ret = flb_parser_do(parser, input, strlen(input), &out_buf, &out_size, &out_time); - if (!TEST_CHECK(ret != -1)) { - TEST_MSG("flb_parser_do failed"); - flb_parser_destroy(parser); - flb_config_exit(config); - exit(1); - } - - ret = compare_msgpack(out_buf, out_size, &expected); - if (!TEST_CHECK(ret == 0)) { - TEST_MSG("compare failed"); - flb_free(out_buf); - flb_parser_destroy(parser); - flb_config_exit(config); - exit(1); - } - - flb_free(out_buf); - flb_parser_destroy(parser); - flb_config_exit(config); -} - - TEST_LIST = { { "basic", test_basic}, { "time_key", test_time_key}, { "time_keep", test_time_keep}, + { "time_key_kept_if_parse_fails", test_time_key_kept_if_parse_fails}, + { "time_use_system_timezone", test_time_use_system_timezone }, { "types_is_not_supported", test_types_is_not_supported}, { "decode_field_json", test_decode_field_json}, - { "time_key_kept_if_parse_fails", test_time_key_kept_if_parse_fails}, { 0 } }; diff --git a/tests/internal/parser_logfmt.c b/tests/internal/parser_logfmt.c index ea363c53ab2..b6ea5bbbf8e 100644 --- a/tests/internal/parser_logfmt.c +++ b/tests/internal/parser_logfmt.c @@ -171,7 +171,7 @@ void test_basic() } parser = flb_parser_create("logfmt", "logfmt", NULL, FLB_FALSE, NULL, NULL, NULL, - FLB_FALSE, FLB_FALSE, FLB_FALSE, + FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE, NULL, 0, NULL, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -223,7 +223,7 @@ void test_time_key() } parser = flb_parser_create("logfmt", "logfmt", NULL, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL, - FLB_FALSE, FLB_FALSE, FLB_FALSE, + FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE, NULL, 0, NULL, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -284,7 +284,7 @@ void test_time_keep() } parser = flb_parser_create("logfmt", "logfmt", NULL, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL, - FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE, + FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE, FLB_FALSE, NULL, 0, NULL, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -319,6 +319,99 @@ void test_time_keep() flb_config_exit(config); } +void test_time_use_system_timezone() +{ + struct flb_parser *parser = NULL; + struct flb_config *config = NULL; + int ret = 0; + char tz[10] = "CET+1CEST\0"; + char *original_tz; + /** + * TZ=CEST date -d @1597104000 + * Tue Aug 11 12:00:00 AM CEST 2020 + * + * TZ=GMT date -d @1597104000 + * Tue Aug 11 12:00:00 AM GMT 2020 + */ + time_t expected_time = 1597104000; + char *input = "str=\"text\" int=100 double=1.23 bool=true time=2020-08-11T00:00:00"; + void *out_buf = NULL; + size_t out_size = 0; + struct flb_time out_time; + char *expected_strs[] = {"str", "text", "int", "100", "double","1.23", "bool", "true"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + out_time.tm.tv_sec = 0; + out_time.tm.tv_nsec = 0; + + + config = flb_config_init(); + if(!TEST_CHECK(config != NULL)) { + TEST_MSG("flb_config_init failed"); + exit(1); + } + + parser = flb_parser_create("logfmt", "logfmt", NULL, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL, + FLB_FALSE, FLB_FALSE, + FLB_TRUE, /* time_system_timezone */ FLB_FALSE, + NULL, 0, NULL, config); + if (!TEST_CHECK(parser != NULL)) { + TEST_MSG("flb_parser_create failed"); + flb_config_exit(config); + exit(1); + } + + /** + * Set the timezone to ensure mktime picks it up. + */ + original_tz = getenv("TZ"); + ret = setenv("TZ", tz, 1); + TEST_ASSERT_(ret == 0, "failed to set timezone to %s", tz); + + ret = flb_parser_do(parser, input, strlen(input), &out_buf, &out_size, &out_time); + if (!TEST_CHECK(ret != -1)) { + TEST_MSG("flb_parser_do failed"); + flb_parser_destroy(parser); + flb_config_exit(config); + exit(1); + } + + ret = compare_msgpack(out_buf, out_size, &expected); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("compare failed"); + flb_free(out_buf); + flb_parser_destroy(parser); + flb_config_exit(config); + exit(1); + } + + if (!TEST_CHECK(out_time.tm.tv_sec == expected_time)) { + TEST_MSG("timestamp error. sec Got=%ld Expect=%"PRIu32"", + out_time.tm.tv_sec, (uint32_t)expected_time); + } + + /** + * Restore original timezone. + */ + if (original_tz) { + ret = setenv("TZ", original_tz, 1); + TEST_ASSERT_(ret == 0, + "failed to restore timezone to %s", original_tz); + flb_free(original_tz); + } else { + ret = unsetenv("TZ"); + TEST_ASSERT_(ret == 0, + "failed to restore original timezone setting"); + } + + flb_free(out_buf); + flb_parser_destroy(parser); + flb_config_exit(config); +} + void test_types() { struct flb_parser *parser = NULL; @@ -361,7 +454,7 @@ void test_types() types->type = FLB_PARSER_TYPE_HEX; parser = flb_parser_create("logfmt", "logfmt", NULL, FLB_FALSE, NULL, NULL, NULL, - FLB_FALSE, FLB_FALSE, FLB_FALSE, + FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE, types, 1, NULL, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -393,6 +486,7 @@ TEST_LIST = { { "basic", test_basic}, { "time_key", test_time_key}, { "time_keep", test_time_keep}, + { "time_use_system_timezone", test_time_use_system_timezone }, { "types", test_types}, { 0 } }; diff --git a/tests/internal/parser_ltsv.c b/tests/internal/parser_ltsv.c index 3f9170518aa..49b44fc45de 100644 --- a/tests/internal/parser_ltsv.c +++ b/tests/internal/parser_ltsv.c @@ -171,7 +171,7 @@ void test_basic() } parser = flb_parser_create("ltsv", "ltsv", NULL, FLB_FALSE, NULL, NULL, NULL, - FLB_FALSE, FLB_FALSE, FLB_FALSE, + FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE, NULL, 0, NULL, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -223,7 +223,7 @@ void test_time_key() } parser = flb_parser_create("ltsv", "ltsv", NULL, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL, - FLB_FALSE, FLB_FALSE, FLB_FALSE, + FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE, NULL, 0, NULL, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -284,7 +284,7 @@ void test_time_keep() } parser = flb_parser_create("ltsv", "ltsv", NULL, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL, - FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE, + FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE, FLB_FALSE, NULL, 0, NULL, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -319,6 +319,99 @@ void test_time_keep() flb_config_exit(config); } +void test_time_use_system_timezone() +{ + struct flb_parser *parser = NULL; + struct flb_config *config = NULL; + int ret = 0; + char tz[10] = "CET+1CEST\0"; + char *original_tz; + /** + * TZ=CEST date -d @1597104000 + * Tue Aug 11 12:00:00 AM CEST 2020 + * + * TZ=GMT date -d @1597104000 + * Tue Aug 11 12:00:00 AM GMT 2020 + */ + time_t expected_time = 1597104000; + char *input = "str:text\tint:100\tdouble:1.23\tbool:true\ttime:2020-08-11T00:00:00"; + void *out_buf = NULL; + size_t out_size = 0; + struct flb_time out_time; + char *expected_strs[] = {"str", "text", "int", "100", "double","1.23", "bool", "true"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + out_time.tm.tv_sec = 0; + out_time.tm.tv_nsec = 0; + + config = flb_config_init(); + if(!TEST_CHECK(config != NULL)) { + TEST_MSG("flb_config_init failed"); + exit(1); + } + + parser = flb_parser_create("ltsv", "ltsv", NULL, FLB_FALSE, "%Y-%m-%dT%H:%M:%S", "time", NULL, + FLB_FALSE, FLB_FALSE, + FLB_TRUE, /* time_system_timezone */ FLB_FALSE, + NULL, 0, NULL, config); + if (!TEST_CHECK(parser != NULL)) { + TEST_MSG("flb_parser_create failed"); + flb_config_exit(config); + exit(1); + } + + /** + * Set the timezone to ensure mktime picks it up. + */ + original_tz = getenv("TZ"); + ret = setenv("TZ", tz, 1); + TEST_ASSERT_(ret == 0, "failed to set timezone to %s", tz); + + ret = flb_parser_do(parser, input, strlen(input), &out_buf, &out_size, &out_time); + if (!TEST_CHECK(ret != -1)) { + TEST_MSG("flb_parser_do failed"); + flb_parser_destroy(parser); + flb_config_exit(config); + exit(1); + } + + ret = compare_msgpack(out_buf, out_size, &expected); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("compare failed"); + flb_free(out_buf); + flb_parser_destroy(parser); + flb_config_exit(config); + exit(1); + } + + if (!TEST_CHECK(out_time.tm.tv_sec == expected_time)) { + TEST_MSG("timestamp error. sec Got=%ld Expect=%"PRIu32"", + out_time.tm.tv_sec, (uint32_t)expected_time); + } + + /** + * Restore original timezone. + */ + if (original_tz) { + ret = setenv("TZ", original_tz, 1); + TEST_ASSERT_(ret == 0, + "failed to restore timezone to %s", original_tz); + flb_free(original_tz); + } else { + ret = unsetenv("TZ"); + TEST_ASSERT_(ret == 0, + "failed to restore original timezone setting"); + } + + flb_free(out_buf); + flb_parser_destroy(parser); + flb_config_exit(config); +} + + void test_types() { struct flb_parser *parser = NULL; @@ -361,7 +454,7 @@ void test_types() types->type = FLB_PARSER_TYPE_HEX; parser = flb_parser_create("ltsv", "ltsv", NULL, FLB_FALSE, NULL, NULL, NULL, - FLB_FALSE, FLB_FALSE, FLB_FALSE, + FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE, types, 1, NULL, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -446,7 +539,7 @@ void test_decode_field_json() } parser = flb_parser_create("ltsv", "ltsv", NULL, FLB_FALSE, NULL, NULL, NULL, - FLB_FALSE, FLB_FALSE, FLB_FALSE, + FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE, NULL, 0, decoder, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -480,6 +573,7 @@ TEST_LIST = { { "basic", test_basic}, { "time_key", test_time_key}, { "time_keep", test_time_keep}, + { "time_use_system_timezone", test_time_use_system_timezone }, { "types", test_types}, { "decode_field_json", test_decode_field_json}, { 0 } diff --git a/tests/internal/parser_regex.c b/tests/internal/parser_regex.c index 2283231ce57..582466cbf42 100644 --- a/tests/internal/parser_regex.c +++ b/tests/internal/parser_regex.c @@ -172,7 +172,7 @@ void test_basic() } parser = flb_parser_create("regex", "regex", regex, FLB_FALSE, NULL, NULL, NULL, - FLB_FALSE, FLB_FALSE, FLB_FALSE, + FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE, NULL, 0, NULL, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -225,7 +225,7 @@ void test_time_key() } parser = flb_parser_create("regex", "regex", regex, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL, - FLB_FALSE, FLB_FALSE, FLB_FALSE, + FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE, NULL, 0, NULL, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -287,7 +287,7 @@ void test_time_keep() } parser = flb_parser_create("regex", "regex", regex, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL, - FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE, + FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE, FLB_FALSE, NULL, 0, NULL, config); if (!TEST_CHECK(parser != NULL)) { TEST_MSG("flb_parser_create failed"); @@ -322,6 +322,100 @@ void test_time_keep() flb_config_exit(config); } +void test_time_use_system_timezone() +{ + struct flb_parser *parser = NULL; + struct flb_config *config = NULL; + int ret = 0; + char tz[10] = "CET+1CEST\0"; + char *original_tz; + /** + * TZ=CEST date -d @1597104000 + * Tue Aug 11 12:00:00 AM CEST 2020 + * + * TZ=GMT date -d @1597104000 + * Tue Aug 11 12:00:00 AM GMT 2020 + */ + time_t expected_time = 1597104000; + char *input = "text 100 1.23 true 2020-08-11T00:00:00"; + char *regex = "(?[a-z]+) (?\\d+) (?\\d+\\.\\d+) (?true) (?