flb_parser: add time_system_timezone

Added a new config option for parsers called `time_system_timezone`
which will fall back to `mktime` for tm to time conversion which will
force the usage of the system timezone.

Signed-off-by: Braydon Kains <[email protected]>

include/fluent-bit/flb_parser.h

@@ -48,6 +48,7 @@ struct flb_parser {
     char *time_fmt_full;  /* original given time format */
     char *time_key;       /* field name that contains the time */
     int time_offset;      /* fixed UTC offset */
+    int time_system_timezone; /* use the system timezone as a fallback */
     int time_keep;        /* keep time field */
     int time_strict;      /* parse time field strictly */
     int logfmt_no_bare_keys; /* in logfmt parsers, require all keys to have values */
@@ -74,13 +75,19 @@ enum {
     FLB_PARSER_TYPE_HEX,
 };
 
-static inline time_t flb_parser_tm2time(const struct flb_tm *src)
+static inline time_t flb_parser_tm2time(const struct flb_tm *src, 
+                                        int use_system_timezone)
 {
     struct tm tmp;
     time_t res;
 
     tmp = src->tm;
-    res = timegm(&tmp) - flb_tm_gmtoff(src);
+    if (use_system_timezone) {
+        tmp.tm_isdst = -1;
+        res = mktime(&tmp);
+    } else {
+        res = timegm(&tmp) - flb_tm_gmtoff(src);
+    }
     return res;
 }
 
@@ -92,6 +99,7 @@ struct flb_parser *flb_parser_create(const char *name, const char *format,
                                      const char *time_offset,
                                      int time_keep,
                                      int time_strict,
+                                     int time_system_timezone,
                                      int logfmt_no_bare_keys,
                                      struct flb_parser_types *types,
                                      int types_len,

plugins/in_kubernetes_events/kubernetes_events.c

@@ -175,7 +175,7 @@ static int timestamp_lookup(struct k8s_events *ctx, char *ts, struct flb_time *t
         return -1;
     }
 
-    time->tm.tv_sec = flb_parser_tm2time(&tm);
+    time->tm.tv_sec = flb_parser_tm2time(&tm, FLB_FALSE);
     time->tm.tv_nsec = 0;
 
     return 0;

src/flb_parser.c

@@ -152,6 +152,7 @@ struct flb_parser *flb_parser_create(const char *name, const char *format,
                                      const char *time_offset,
                                      int time_keep,
                                      int time_strict,
+                                     int time_system_timezone,
                                      int logfmt_no_bare_keys,
                                      struct flb_parser_types *types,
                                      int types_len,
@@ -312,8 +313,17 @@ struct flb_parser *flb_parser_create(const char *name, const char *format,
             p->time_frac_secs = (tmp + 2);
         }
 
-        /* Optional fixed timezone offset */
-        if (time_offset) {
+        /* 
+         * Fall back to the system timezone 
+         * if there is no zone parsed from the log.  
+         */
+        p->time_system_timezone = time_system_timezone;
+
+        /* 
+         * Optional fixed timezone offset, only applied if 
+         * not falling back to system timezone. 
+         */
+        if (!p->time_system_timezone && time_offset) {
             diff = 0;
             len = strlen(time_offset);
             ret = flb_parser_tzone_offset(time_offset, len, &diff);
@@ -487,6 +497,7 @@ static int parser_conf_file(const char *cfg, struct flb_cf *cf,
     int skip_empty;
     int time_keep;
     int time_strict;
+    int time_system_timezone;
     int logfmt_no_bare_keys;
     int types_len;
     struct mk_list *head;
@@ -561,6 +572,13 @@ static int parser_conf_file(const char *cfg, struct flb_cf *cf,
             flb_sds_destroy(tmp_str);
         }
 
+        time_system_timezone = FLB_FALSE;
+        tmp_str = get_parser_key(config, cf, s, "time_system_timezone");
+        if (tmp_str) {
+            time_system_timezone = flb_utils_bool(tmp_str);
+            flb_sds_destroy(tmp_str);
+        }
+
         /* time_offset (UTC offset) */
         time_offset = get_parser_key(config, cf, s, "time_offset");
 
@@ -587,7 +605,8 @@ static int parser_conf_file(const char *cfg, struct flb_cf *cf,
         /* Create the parser context */
         if (!flb_parser_create(name, format, regex, skip_empty,
                                time_fmt, time_key, time_offset, time_keep, time_strict,
-                               logfmt_no_bare_keys, types, types_len, decoders, config)) {
+                               time_system_timezone, logfmt_no_bare_keys, types, types_len, 
+                               decoders, config)) {
             goto fconf_error;
         }
 

src/flb_parser_json.c

@@ -207,7 +207,7 @@ int flb_parser_json_do(struct flb_parser *parser,
         skip = map_size;
     }
     else {
-        time_lookup = flb_parser_tm2time(&tm);
+        time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone);
     }
 
     /* Compose a new map without the time_key field */

src/flb_parser_logfmt.c

@@ -166,7 +166,7 @@ static int logfmt_parser(struct flb_parser *parser,
                                   parser->name, parser->time_fmt_full);
                         return -1;
                     }
-                    *time_lookup = flb_parser_tm2time(&tm);
+                    *time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone);
                 }
                 time_found = FLB_TRUE;
             }

src/flb_parser_ltsv.c

@@ -139,7 +139,7 @@ static int ltsv_parser(struct flb_parser *parser,
                                  parser->name, parser->time_fmt_full);
                        return -1;
                     }
-                    *time_lookup = flb_parser_tm2time(&tm);
+                    *time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone);
                 }
                 time_found = FLB_TRUE;
             }

src/flb_parser_regex.c

@@ -87,7 +87,7 @@ static void cb_results(const char *name, const char *value,
             }
 
             pcb->time_frac = frac;
-            pcb->time_lookup = flb_parser_tm2time(&tm);
+            pcb->time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone);
 
             if (parser->time_keep == FLB_FALSE) {
                 pcb->num_skipped++;

src/multiline/flb_ml_parser_cri.c

@@ -40,6 +40,7 @@ static struct flb_parser *cri_parser_create(struct flb_config *config)
                           NULL,                    /* time offset */
                           FLB_TRUE,                /* time keep */
                           FLB_FALSE,               /* time strict */
+                          FLB_FALSE,               /* time system timezone */
                           FLB_FALSE,               /* no bare keys */
                           NULL,                    /* parser types */
                           0,                       /* types len */

src/multiline/flb_ml_parser_docker.c

@@ -35,6 +35,7 @@ static struct flb_parser *docker_parser_create(struct flb_config *config)
                           NULL,                   /* time offset */
                           FLB_TRUE,               /* time keep */
                           FLB_FALSE,              /* time strict */
+                          FLB_FALSE,              /* time system timezone */
                           FLB_FALSE,              /* no bare keys */
                           NULL,                   /* parser types */
                           0,                      /* types len */

tests/internal/parser.c

@@ -124,7 +124,7 @@ int flb_parser_regex_do(struct flb_parser *parser,
                         struct flb_time *out_time);
 
 /* Parse timezone string and get the offset */
-void test_parser_tzone_offset()
+void test_parser_tzone_offset(void)
 {
     int i;
     int len;
@@ -167,7 +167,7 @@ static void load_regex_parsers(struct flb_config *config)
     TEST_CHECK(ret == 0);
 }
 
-void test_parser_time_lookup()
+void test_parser_time_lookup(void)
 {
     int i;
     int j;
@@ -231,7 +231,7 @@ void test_parser_time_lookup()
             continue;
         }
 
-        epoch = flb_parser_tm2time(&tm);
+        epoch = flb_parser_tm2time(&tm, FLB_FALSE);
         epoch -= year_diff;
         TEST_CHECK(t->epoch == epoch);
         TEST_CHECK(t->frac_seconds == ns);
@@ -246,7 +246,7 @@ void test_parser_time_lookup()
 }
 
 /* Do time lookup using the JSON parser backend*/
-void test_json_parser_time_lookup()
+void test_json_parser_time_lookup(void)
 {
     int i;
     int j;
@@ -331,7 +331,7 @@ void test_json_parser_time_lookup()
 }
 
 /* Do time lookup using the Regex parser backend*/
-void test_regex_parser_time_lookup()
+void test_regex_parser_time_lookup(void)
 {
     int i;
     int j;
@@ -487,8 +487,7 @@ static int a_mysql_unquote_test(struct flb_parser *p, char *source, char *expect
     return 1;
 }
 
-
-void test_mysql_unquoted()
+void test_mysql_unquoted(void)
 {
     struct flb_parser *p;
     struct flb_config *config;
@@ -513,16 +512,183 @@ void test_mysql_unquoted()
 
     flb_parser_exit(config);
     flb_config_exit(config);
+}
+
+/**
+ * I've opted to leave this function around for future testers.
+ * When working with these C time structures, it can really help
+ * to just sanity check what the timestamp actually is under the
+ * hood. If you are adding or working with the timestamp tests,
+ * you can manually add this function to print out a human readable
+ * timestamp. Note that it will be in your system's local timezone;
+ * you'll have to do some math based on the test you're working with.
+ * - Braydon Kains, @braydonk
+ */
+void debug_print_local_timestamp(time_t t)
+{
+    struct tm tm;
+    char buf[80];
+    tm = *localtime(&t);
+    strftime(buf, sizeof(buf), "%a %Y-%m-%d %H:%M:%S %Z", &tm);
+    printf("%s\n", buf);
+}
+
+/**
+ * Run tm2time on a flb_tm and expect a certain epoch (in seconds).
+ */
+void tm2time_testcase(struct flb_tm *tm, time_t expected_epoch, int system_timezone)
+{
+    time_t result_epoch = flb_parser_tm2time(tm, system_timezone);
+    TEST_CHECK_(result_epoch == expected_epoch, 
+                "expected (%lld), got (%lld)", 
+                (long long)expected_epoch, (long long)result_epoch);
+}
+
+/**
+ * Perform a tm2time_testcase, but first set the specified timezone, and
+ * reset the timezone after.
+ */
+void set_tz_tm2time_testcase(char *tz, struct flb_tm *tm, time_t expected_epoch)
+{
+    int ret;
+    char *original_tz = NULL;
+
+    /* Set the timezone and keep the original value to reset it later. */
+    if (tz) {
+        original_tz = getenv("TZ");
+        ret = setenv("TZ", tz, 1);
+        TEST_ASSERT_(ret == 0, 
+                     "failed to set timezone to %s", tz);
+    }
+
+    tm2time_testcase(tm, expected_epoch, FLB_TRUE);
+
+    /* Reset the timezone. */
+    if (original_tz) {
+        ret = setenv("TZ", original_tz, 1);
+        TEST_ASSERT_(ret == 0, 
+                     "failed to restore timezone to %s", original_tz);
+        flb_free(original_tz);
+    } else {
+        ret = unsetenv("TZ");
+        TEST_ASSERT_(ret == 0,
+                     "failed to restore original timezone setting");
+    }
+}
+
+void test_tm2time_timegm(void)
+{
+    /**
+     * TZ=GMT date -d @1673654400
+     * Mon Feb 20 12:00:00 AM EST 2023
+     */
+    time_t expected_epoch = 1673654400;
+    struct flb_tm *tm = malloc(sizeof(struct tm)); 
 
+    tm->tm.tm_year = 2023-1900;
+    tm->tm.tm_mon = 0;
+    tm->tm.tm_mday = 14;
+    tm->tm.tm_hour = 0;
+    tm->tm.tm_min = 0;
+    tm->tm.tm_sec = 0;
 
+    tm2time_testcase(tm, expected_epoch, FLB_FALSE);
+
+    flb_free(tm);
+}
+
+void test_tm2time_timegm_with_gmtoff(void)
+{
+    /**
+     * TZ=GMT date -d @1676869200
+     * Mon Feb 20 05:00:00 AM GMT 2023
+     *
+     * TZ=EST date -d @1676869200
+     * Mon Feb 20 12:00:00 AM EST 2023
+     */
+    time_t expected_epoch = 1676869200;
+    struct flb_tm *tm = malloc(sizeof(struct flb_tm));
+
+    tm->tm.tm_year = 2023-1900;
+    tm->tm.tm_mon = 1;
+    tm->tm.tm_mday = 20;
+    tm->tm.tm_hour = 0;
+    tm->tm.tm_min = 0;
+    tm->tm.tm_sec = 0;
+    /**
+     * tm_gmtoff is the number of seconds east of GMT.
+     * This calculation equates to UTC-5, which in January
+     * is EST.
+     */
+    flb_tm_gmtoff(tm) = (-5)*3600;
+
+    tm2time_testcase(tm, expected_epoch, FLB_FALSE);
+
+    flb_free(tm);
+}
+
+void test_tm2time_system_timezone(void)
+{
+    /**
+     * TZ=MST7MDT date -d @1676876400
+     * Mon Feb 20 12:00:00 AM MST 2023
+     *
+     * TZ=GMT date -d @1676876400
+     * Mon Feb 20 07:00:00 AM GMT 2023
+     */
+    time_t expected_epoch = 1676876400;
+    struct flb_tm *tm = malloc(sizeof(struct flb_tm));
+
+    tm->tm.tm_year = 2023-1900;
+    tm->tm.tm_mon = 1;
+    tm->tm.tm_mday = 20;
+    tm->tm.tm_hour = 0;
+    tm->tm.tm_min = 0;
+    tm->tm.tm_sec = 0;
+
+    set_tz_tm2time_testcase("MST7MDT", tm, expected_epoch);
+
+    flb_free(tm);
 }
 
+void test_tm2time_mktime_system_timezone_find_dst(void)
+{
+    /**
+     * TZ=GMT date -d @1587355200
+     * Mon Apr 20 04:00:00 AM GMT 2020
+     *
+     * TZ=EST5EDT date -d @1587355200
+     * Mon Apr 20 12:00:00 AM EDT 2020
+     */
+    time_t expected_epoch = 1587355200;
+    struct flb_tm *tm = malloc(sizeof(struct flb_tm));
+
+    tm->tm.tm_year = 2020-1900;
+    tm->tm.tm_mon = 3;
+    tm->tm.tm_mday = 20;
+    tm->tm.tm_hour = 0;
+    tm->tm.tm_min = 0;
+    tm->tm.tm_sec = 0;
+
+    /**
+     * In this test case, tm2time should be able automatically
+     * calculating and determining EDT is correct based on the 
+     * date of April 20, which is after the daylight savings switch. 
+     */ 
+    set_tz_tm2time_testcase("EST5EDT", tm, expected_epoch);
+
+    flb_free(tm);
+}
 
 TEST_LIST = {
     { "tzone_offset", test_parser_tzone_offset},
     { "time_lookup", test_parser_time_lookup},
     { "json_time_lookup", test_json_parser_time_lookup},
     { "regex_time_lookup", test_regex_parser_time_lookup},
-    { "mysql_unquoted" , test_mysql_unquoted },
+    { "mysql_unquoted", test_mysql_unquoted },
+    { "tm2time_timegm", test_tm2time_timegm },
+    { "tm2time_timegm_with_gmtoff", test_tm2time_timegm_with_gmtoff },
+    { "tm2time_system_timezone", test_tm2time_system_timezone },
+    { "tm2time_system_timezone_find_dst", test_tm2time_mktime_system_timezone_find_dst },
     { 0 }
 };