filter_lua: support enable_flb_null

include/fluent-bit/flb_lua.h

@@ -29,6 +29,9 @@
 #include <monkey/mk_core/mk_list.h>
 #include <msgpack/pack.h>
 
+/* global variables in Lua */
+#define FLB_LUA_VAR_FLB_NULL "flb_null"
+
 #define FLB_LUA_L2C_TYPES_NUM_MAX   16
 
 enum flb_lua_l2c_type_enum {
@@ -74,5 +77,6 @@ void flb_lua_tompack(lua_State *l,
                      int index,
                      struct flb_lua_l2c_config *l2cc);
 void flb_lua_dump_stack(FILE *out, lua_State *l);
+int flb_lua_enable_flb_null(lua_State *l);
 
 #endif

plugins/filter_lua/lua.c

@@ -61,6 +61,10 @@ static int cb_lua_init(struct flb_filter_instance *f_ins,
     }
     ctx->lua = lj;
 
+    if (ctx->enable_flb_null) {
+        flb_lua_enable_flb_null(lj->state);
+    }
+
     /* Lua script source code */
     if (ctx->code) {
         ret = flb_luajit_load_buffer(ctx->lua,
@@ -683,6 +687,13 @@ static struct flb_config_map config_map[] = {
      "If enabled, Fluent-bit will pass the timestamp as a Lua table "
      "with keys \"sec\" for seconds since epoch and \"nsec\" for nanoseconds."
     },
+    {
+     FLB_CONFIG_MAP_BOOL, "enable_flb_null", "false",
+     0, FLB_TRUE, offsetof(struct lua_filter, enable_flb_null),
+     "If enabled, null will be converted to flb_null in Lua. "
+     "It is useful to prevent removing key/value "
+     "since nil is a special value to remove key value from map in Lua."
+    },
 
     {0}
 };

plugins/filter_lua/lua_config.h

@@ -35,6 +35,7 @@ struct lua_filter {
     flb_sds_t buffer;                 /* json dec buffer */
     int    protected_mode;            /* exec lua function in protected mode */
     int    time_as_table;             /* timestamp as a Lua table */
+    int    enable_flb_null;           /* Use flb_null in Lua */
     struct flb_lua_l2c_config l2cc;   /* lua -> C config */
     struct flb_luajit *lua;           /* state context   */
     struct flb_filter_instance *ins;  /* filter instance */

src/flb_lua.c

@@ -25,6 +25,17 @@
 #include <fluent-bit/flb_lua.h>
 #include <stdint.h>
 
+int flb_lua_enable_flb_null(lua_State *l)
+{
+    /* set flb.null */
+    lua_pushlightuserdata(l, NULL);
+
+    flb_info("[%s] set %s", __FUNCTION__, FLB_LUA_VAR_FLB_NULL);
+    lua_setglobal(l, FLB_LUA_VAR_FLB_NULL);
+
+    return 0;
+}
+
 void flb_lua_pushtimetable(lua_State *l, struct flb_time *tm)
 {
     lua_createtable(l, 0, 2);
@@ -63,7 +74,7 @@ int flb_lua_pushmpack(lua_State *l, mpack_reader_t *reader)
     tag = mpack_read_tag(reader);
     switch (mpack_tag_type(&tag)) {
         case mpack_type_nil:
-            lua_pushnil(l);
+            lua_getglobal(l, FLB_LUA_VAR_FLB_NULL);
             break;
         case mpack_type_bool:
             lua_pushboolean(l, mpack_tag_bool_value(&tag));
@@ -128,7 +139,7 @@ void flb_lua_pushmsgpack(lua_State *l, msgpack_object *o)
 
     switch(o->type) {
         case MSGPACK_OBJECT_NIL:
-            lua_pushnil(l);
+            lua_getglobal(l, FLB_LUA_VAR_FLB_NULL);
             break;
 
         case MSGPACK_OBJECT_BOOLEAN:

tests/runtime/filter_lua.c

@@ -677,6 +677,76 @@ void flb_test_drop_all_records(void)
     flb_destroy(ctx);
 }
 
+void flb_test_enable_flb_null(void)
+{
+    int ret;
+    flb_ctx_t *ctx;
+    int in_ffd;
+    int out_ffd;
+    int filter_ffd;
+    char *output = NULL;
+    char *input = "[0, {\"hello\":null}]";
+    char *result;
+    struct flb_lib_out_cb cb_data;
+
+    char *script_body = ""
+      "function lua_main(tag, timestamp, record)\n"
+      "    return 1, timestamp, record\n"
+      "end\n";
+
+    clear_output();
+
+    /* Create context, flush every second (some checks omitted here) */
+    ctx = flb_create();
+    flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL);
+
+    /* Prepare output callback context*/
+    cb_data.cb = callback_test;
+    cb_data.data = NULL;
+
+    ret = create_script(script_body, strlen(script_body));
+    TEST_CHECK(ret == 0);
+    /* Filter */
+    filter_ffd = flb_filter(ctx, (char *) "lua", NULL);
+    TEST_CHECK(filter_ffd >= 0);
+    ret = flb_filter_set(ctx, filter_ffd,
+                         "Match", "*",
+                         "call", "lua_main",
+                         "script", TMP_LUA_PATH,
+                         "enable_flb_null", "true",
+                         NULL);
+
+    /* Input */
+    in_ffd = flb_input(ctx, (char *) "lib", NULL);
+    flb_input_set(ctx, in_ffd, "tag", "test", NULL);
+    TEST_CHECK(in_ffd >= 0);
+
+    /* Lib output */
+    out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data);
+    TEST_CHECK(out_ffd >= 0);
+    flb_output_set(ctx, out_ffd,
+                   "match", "test",
+                   "format", "json",
+                   NULL);
+
+    ret = flb_start(ctx);
+    TEST_CHECK(ret==0);
+
+    flb_lib_push(ctx, in_ffd, input, strlen(input));
+    wait_with_timeout(2000, &output);
+    result = strstr(output, "\"hello\":null");
+    if(!TEST_CHECK(result != NULL)) {
+        TEST_MSG("output:%s\n", output);
+    }
+
+    /* clean up */
+    flb_lib_free(output);
+    delete_script();
+
+    flb_stop(ctx);
+    flb_destroy(ctx);
+}
+
 /* https://github.com/fluent/fluent-bit/issues/5496 */
 void flb_test_split_record(void)
 {
@@ -761,6 +831,7 @@ TEST_LIST = {
     {"type_array_key", flb_test_type_array_key},
     {"array_contains_null", flb_test_array_contains_null},
     {"drop_all_records", flb_test_drop_all_records},
+    {"enable_flb_null", flb_test_enable_flb_null},
     {"split_record", flb_test_split_record},
     {NULL, NULL}
 };