Merge branch 'fluent:master' into master

CMakeLists.txt

@@ -7,8 +7,8 @@ set(CMAKE_POLICY_DEFAULT_CMP0069 NEW)
 
 # Fluent Bit Version
 set(FLB_VERSION_MAJOR  2)
-set(FLB_VERSION_MINOR  1)
-set(FLB_VERSION_PATCH  10)
+set(FLB_VERSION_MINOR  2)
+set(FLB_VERSION_PATCH  0)
 set(FLB_VERSION_STR "${FLB_VERSION_MAJOR}.${FLB_VERSION_MINOR}.${FLB_VERSION_PATCH}")
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)

dockerfiles/Dockerfile

@@ -11,7 +11,7 @@
 # docker buildx build --platform "linux/amd64,linux/arm64,linux/arm/v7" -f ./dockerfiles/Dockerfile.multiarch --build-arg FLB_TARBALL=https://github.com/fluent/fluent-bit/archive/v1.8.11.tar.gz ./dockerfiles/
 
 # Set this to the current release version: it gets done so as part of the release.
-ARG RELEASE_VERSION=2.1.10
+ARG RELEASE_VERSION=2.2.0
 
 # For multi-arch builds - assumption is running on an AMD64 host
 FROM multiarch/qemu-user-static:x86_64-arm as qemu-arm32

fluent-bit-2.2.0.bb

@@ -0,0 +1,57 @@
+# Fluent Bit - Yocto / Bitbake
+# ============================
+# The following Bitbake package the latest Fluent Bit stable release.
+
+SUMMARY = "Fast Log processor and Forwarder"
+DESCRIPTION = "Fluent Bit is a data collector, processor and  \
+forwarder for Linux. It supports several input sources and \
+backends (destinations) for your data. \
+"
+
+HOMEPAGE = "http://fluentbit.io"
+BUGTRACKER = "https://github.com/fluent/fluent-bit/issues"
+
+LICENSE = "Apache-2.0"
+LIC_FILES_CHKSUM = "file://LICENSE;md5=2ee41112a44fe7014dce33e26468ba93"
+SECTION = "net"
+
+PR = "r0"
+PV = "2.2.0"
+
+SRCREV = "v${PV}"
+SRC_URI = "git://github.com/fluent/fluent-bit.git;nobranch=1"
+
+S = "${WORKDIR}/git"
+DEPENDS = "zlib bison-native flex-native"
+INSANE_SKIP_${PN}-dev += "dev-elf"
+
+# Use CMake 'Unix Makefiles' generator
+OECMAKE_GENERATOR ?= "Unix Makefiles"
+
+# Fluent Bit build options
+# ========================
+
+# Host related setup
+EXTRA_OECMAKE += "-DGNU_HOST=${HOST_SYS} "
+
+# Disable LuaJIT and filter_lua support
+EXTRA_OECMAKE += "-DFLB_LUAJIT=Off -DFLB_FILTER_LUA=Off "
+
+# Disable Library and examples
+EXTRA_OECMAKE += "-DFLB_SHARED_LIB=Off -DFLB_EXAMPLES=Off "
+
+# Systemd support (optional)
+DEPENDS += "systemd"
+EXTRA_OECMAKE += "-DFLB_IN_SYSTEMD=On "
+
+# Kafka Output plugin (disabled by default): note that when
+# enabling Kafka output plugin, the backend library librdkafka
+# requires 'openssl' as a dependency.
+#
+# DEPENDS += "openssl "
+# EXTRA_OECMAKE += "-DFLB_OUT_KAFKA=On "
+
+inherit cmake systemd
+
+SYSTEMD_SERVICE_${PN} = "fluent-bit.service"
+TARGET_CC_ARCH_append = " ${SELECTED_OPTIMIZATION}"

include/fluent-bit/flb_log_event_encoder.h

@@ -28,6 +28,43 @@
 
 #include <msgpack.h>
 
+/* Note:
+ *       64 bit Windows :
+ *              All of the size_t casts have been replaced with
+ *              size_t pointer casts because there is an issue
+ *              with msvc where it doesn't honor the type promotion
+ *              when a small constant integer is hard-coded.
+ *
+ *              This should not be a problem because according to
+ *              the standard a size_t should be as large as the
+ *              native register size just like pointers.
+ *
+ *       32 bit Windows :
+ *              64 bit integers are still defined as their specific
+ *              types because the compiler handles them properly.
+ *
+ *       Additionally, even though it would be preferrable to be
+ *       able to use the optimize pragma to selectively disable
+ *       the problematic optimizations it doesn't seem to work
+ *       as expected.
+ */
+
+#ifdef FLB_SYSTEM_WINDOWS
+#ifdef _WIN64
+typedef size_t * flb_log_event_encoder_size_t;
+typedef size_t * flb_log_event_encoder_int64_t;
+typedef size_t * flb_log_event_encoder_uint64_t;
+#else
+typedef size_t * flb_log_event_encoder_size_t;
+typedef int64_t  flb_log_event_encoder_int64_t;
+typedef uint64_t flb_log_event_encoder_uint64_t;
+#endif
+#else
+typedef size_t   flb_log_event_encoder_size_t;
+typedef int64_t  flb_log_event_encoder_int64_t;
+typedef uint64_t flb_log_event_encoder_uint64_t;
+#endif
+
 #define FLB_EVENT_ENCODER_SUCCESS                        0
 #define FLB_EVENT_ENCODER_ERROR_UNSPECIFIED             -1
 #define FLB_EVENT_ENCODER_ERROR_ALLOCATION_ERROR        -2
@@ -80,31 +117,31 @@
 
 #define FLB_LOG_EVENT_STRING_LENGTH_VALUE(length) \
             (int) FLB_LOG_EVENT_STRING_LENGTH_VALUE_TYPE, \
-            (size_t) length
+            (flb_log_event_encoder_size_t) length
 
 #define FLB_LOG_EVENT_STRING_BODY_VALUE(buffer, length) \
             (int) FLB_LOG_EVENT_STRING_BODY_VALUE_TYPE, \
             (char *) buffer, \
-            (size_t) length
+            (flb_log_event_encoder_size_t) length
 
 #define FLB_LOG_EVENT_BINARY_LENGTH_VALUE(length) \
             (int) FLB_LOG_EVENT_BINARY_LENGTH_VALUE_TYPE, \
-            (size_t) length
+            (flb_log_event_encoder_size_t) length
 
 #define FLB_LOG_EVENT_BINARY_BODY_VALUE(buffer, length) \
             (int) FLB_LOG_EVENT_BINARY_BODY_VALUE_TYPE, \
             (char *) buffer, \
-            (size_t) length
+            (flb_log_event_encoder_size_t) length
 
 #define FLB_LOG_EVENT_EXT_LENGTH_VALUE(type_, length) \
             (int) FLB_LOG_EVENT_EXT_LENGTH_VALUE_TYPE, \
             (int) type_, \
-            (size_t) length
+            (flb_log_event_encoder_size_t) length
 
 #define FLB_LOG_EVENT_EXT_BODY_VALUE(buffer, length) \
             (int) FLB_LOG_EVENT_EXT_BODY_VALUE_TYPE, \
             (char *) buffer, \
-            (size_t) length
+            (flb_log_event_encoder_size_t) length
 
 #define FLB_LOG_EVENT_TIMESTAMP_VALUE(value) \
             (int) FLB_LOG_EVENT_TIMESTAMP_VALUE_TYPE, \
@@ -143,7 +180,7 @@
 
 #define FLB_LOG_EVENT_INT64_VALUE(value) \
             (int) FLB_LOG_EVENT_INT64_VALUE_TYPE, \
-            (int64_t) value
+            (flb_log_event_encoder_int64_t) value
 
 #define FLB_LOG_EVENT_UINT8_VALUE(value) \
             (int) FLB_LOG_EVENT_UINT8_VALUE_TYPE, \
@@ -159,7 +196,7 @@
 
 #define FLB_LOG_EVENT_UINT64_VALUE(value) \
             (int) FLB_LOG_EVENT_UINT64_VALUE_TYPE, \
-            (uint64_t) value
+            (flb_log_event_encoder_uint64_t) value
 
 #define FLB_LOG_EVENT_DOUBLE_VALUE(value) \
             (int) FLB_LOG_EVENT_DOUBLE_VALUE_TYPE, \
@@ -176,7 +213,7 @@
 #define FLB_LOG_EVENT_MSGPACK_RAW_VALUE(buffer, length) \
             (int) FLB_LOG_EVENT_MSGPACK_RAW_VALUE_TYPE, \
             (char *) buffer, \
-            (size_t) length
+            (flb_log_event_encoder_size_t) length
 
 #define FLB_LOG_EVENT_STRING_VALUE(buffer, length) \
             FLB_LOG_EVENT_STRING_LENGTH_VALUE(length), \

plugins/filter_log_to_metrics/log_to_metrics.c

@@ -268,7 +268,7 @@ static int set_labels(struct log_to_metrics_ctx *ctx,
             snprintf(label_keys[counter], MAX_LABEL_LENGTH - 1, "%s", kv->val);
             counter++;
         }
-        else if (strcasecmp(kv->key, "label") == 0) {
+        else if (strcasecmp(kv->key, "add_label") == 0) {
             split = flb_utils_split(kv->val, ' ', 1);
             if (mk_list_size(split) != 2) {
                 flb_plg_error(ctx->ins, "invalid label, expected name and key");

src/config_format/flb_cf_yaml.c

@@ -2039,7 +2039,14 @@ static int read_config(struct flb_cf *conf, struct local_ctx *ctx,
     }
 
     yaml_parser_delete(&parser);
-    state_pop(ctx);
+
+    /* free all remaining states */
+    if (code == -1) {
+        while (state = state_pop(ctx));
+    }
+    else {
+        state = state_pop(ctx);
+    }
 
     fclose(fh);
     ctx->level--;

tests/internal/fuzzers/fstore_fuzzer.c

@@ -45,8 +45,25 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
     struct flb_fstore_stream *st;
     struct flb_fstore_file *fsf;
 
+    /* Set flb_malloc_mod to be fuzzer-data dependent */
+    if (size < 4) {
+        return 0;
+    }
+    flb_malloc_p = 0;
+    flb_malloc_mod = *(int*)data;
+    data += 4;
+    size -= 4;
+
+    /* Avoid division by zero for modulo operations */
+    if (flb_malloc_mod == 0) {
+        flb_malloc_mod = 1;
+    }
+
     cio_utils_recursive_delete(FSF_STORE_PATH);
     fs = flb_fstore_create(FSF_STORE_PATH, FLB_FSTORE_FS);
+    if (fs == NULL) {
+        return 0;
+    }
     st = flb_fstore_stream_create(fs, "abc");
     if (st != NULL) {
         fsf = flb_fstore_file_create(fs, st, "example.txt", size);

tests/internal/stream_processor.c

@@ -18,6 +18,7 @@
  *  limitations under the License.
  */
 
+#include <fluent-bit.h>
 #include <fluent-bit/flb_info.h>
 #include <fluent-bit/flb_mem.h>
 #include <fluent-bit/flb_pack.h>
@@ -200,6 +201,8 @@ static void invalid_queries()
     struct flb_sp *sp;
     struct flb_sp_task *task;
 
+    flb_init_env();
+
     /* Total number of checks for invalid queries */
     checks = sizeof(invalid_query_checks) / sizeof(char *);
 
@@ -244,6 +247,8 @@ static void test_select_keys()
     WSADATA wsa_data;
 #endif
 
+    flb_init_env();
+
     config = flb_calloc(1, sizeof(struct flb_config));
     if (!config) {
         flb_errno();
@@ -330,6 +335,8 @@ static void test_select_subkeys()
     WSADATA wsa_data;
 #endif
 
+    flb_init_env();
+
     config = flb_calloc(1, sizeof(struct flb_config));
     if (!config) {
         flb_errno();
@@ -458,6 +465,8 @@ static void test_window()
     WSADATA wsa_data;
 #endif
 
+    flb_init_env();
+
     config = flb_calloc(1, sizeof(struct flb_config));
     if (!config) {
         flb_errno();
@@ -503,6 +512,13 @@ static void test_window()
 
             /* We ingest the buffer every second */
             for (t = 0; t < check->window_size_sec; t++) {
+                if (out_buf.buffer != NULL) {
+                    flb_free(out_buf.buffer);
+
+                    out_buf.buffer = NULL;
+                    out_buf.size = 0;
+                }
+
                 ret = flb_sp_do_test(sp, task,
                                      "samples", strlen("samples"),
                                      &data_buf, &out_buf);
@@ -517,6 +533,13 @@ static void test_window()
                 usleep(800000);
             }
 
+            if (out_buf.buffer != NULL) {
+                flb_free(out_buf.buffer);
+
+                out_buf.buffer = NULL;
+                out_buf.size = 0;
+            }
+
             flb_sp_fd_event_test(task->window.fd, task, &out_buf);
 
             flb_info("[sp test] id=%i, SQL => '%s'", check->id, check->exec);
@@ -560,13 +583,26 @@ static void test_window()
 
                 /* Hopping event */
                 if ((t + 1) % check->window_hop_sec == 0) {
+                    if (out_buf.buffer != NULL) {
+                        flb_free(out_buf.buffer);
+
+                        out_buf.buffer = NULL;
+                        out_buf.size = 0;
+                    }
+
                     flb_sp_fd_event_test(task->window.fd_hop, task, &out_buf);
                 }
 
                 /* Window event */
                 if ((t + 1) % check->window_size_sec == 0 ||
                     (t + 1 > check->window_size_sec && (t + 1 - check->window_size_sec) % check->window_hop_sec == 0)) {
-                    flb_free(out_buf.buffer);
+                    if (out_buf.buffer != NULL) {
+                        flb_free(out_buf.buffer);
+
+                        out_buf.buffer = NULL;
+                        out_buf.size = 0;
+                    }
+
                     flb_sp_fd_event_test(task->window.fd, task, &out_buf);
                 }
                 flb_free(data_buf.buffer);
@@ -612,6 +648,8 @@ static void test_snapshot()
     WSADATA wsa_data;
 #endif
 
+    flb_init_env();
+
     config = flb_calloc(1, sizeof(struct flb_config));
     if (!config) {
         flb_errno();
@@ -773,6 +811,8 @@ static void test_conv_from_str_to_num()
 #endif
     out_buf.buffer = NULL;
 
+    flb_init_env();
+
     config = flb_config_init();
     config->evl = mk_event_loop_create(256);
 

tests/runtime/filter_log_to_metrics.c

@@ -669,7 +669,7 @@ void flb_test_log_to_metrics_label(void)
                          "metric_name", "test",
                          "metric_description", "Counts messages",
                          "kubernetes_mode", "off",
-                         "label", "pod_name $kubernetes['pod_name']",
+                         "add_label", "pod_name $kubernetes['pod_name']",
                          NULL);
 
     out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data);