From 758da4d3801cbc32790f6f212a330ade50d97fac Mon Sep 17 00:00:00 2001 From: Takahiro Yamashita Date: Sat, 9 Sep 2023 07:39:26 +0900 Subject: [PATCH 1/3] lua: push flb_null as a nil value Signed-off-by: Takahiro Yamashita --- include/fluent-bit/flb_lua.h | 4 ++++ src/flb_lua.c | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/include/fluent-bit/flb_lua.h b/include/fluent-bit/flb_lua.h index b2787430284..a578b00deae 100644 --- a/include/fluent-bit/flb_lua.h +++ b/include/fluent-bit/flb_lua.h @@ -29,6 +29,9 @@ #include #include +/* global variables in Lua */ +#define FLB_LUA_VAR_FLB_NULL "flb_null" + #define FLB_LUA_L2C_TYPES_NUM_MAX 16 enum flb_lua_l2c_type_enum { @@ -74,5 +77,6 @@ void flb_lua_tompack(lua_State *l, int index, struct flb_lua_l2c_config *l2cc); void flb_lua_dump_stack(FILE *out, lua_State *l); +int flb_lua_enable_flb_null(lua_State *l); #endif diff --git a/src/flb_lua.c b/src/flb_lua.c index a374752326c..3b3af4926b8 100644 --- a/src/flb_lua.c +++ b/src/flb_lua.c @@ -25,6 +25,17 @@ #include #include +int flb_lua_enable_flb_null(lua_State *l) +{ + /* set flb.null */ + lua_pushlightuserdata(l, NULL); + + flb_info("[%s] set %s", __FUNCTION__, FLB_LUA_VAR_FLB_NULL); + lua_setglobal(l, FLB_LUA_VAR_FLB_NULL); + + return 0; +} + void flb_lua_pushtimetable(lua_State *l, struct flb_time *tm) { lua_createtable(l, 0, 2); @@ -63,7 +74,7 @@ int flb_lua_pushmpack(lua_State *l, mpack_reader_t *reader) tag = mpack_read_tag(reader); switch (mpack_tag_type(&tag)) { case mpack_type_nil: - lua_pushnil(l); + lua_getglobal(l, FLB_LUA_VAR_FLB_NULL); break; case mpack_type_bool: lua_pushboolean(l, mpack_tag_bool_value(&tag)); @@ -128,7 +139,7 @@ void flb_lua_pushmsgpack(lua_State *l, msgpack_object *o) switch(o->type) { case MSGPACK_OBJECT_NIL: - lua_pushnil(l); + lua_getglobal(l, FLB_LUA_VAR_FLB_NULL); break; case MSGPACK_OBJECT_BOOLEAN: From b5cb732ead45a24a5ce108c3caf4087c428ce92d Mon Sep 17 00:00:00 2001 From: Takahiro Yamashita Date: Sat, 9 Sep 2023 07:39:53 +0900 Subject: [PATCH 2/3] tests: runtime: lua: test for enable_flb_null Signed-off-by: Takahiro Yamashita --- tests/runtime/filter_lua.c | 71 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tests/runtime/filter_lua.c b/tests/runtime/filter_lua.c index 9f694377eae..49f64eb8e0c 100644 --- a/tests/runtime/filter_lua.c +++ b/tests/runtime/filter_lua.c @@ -677,6 +677,76 @@ void flb_test_drop_all_records(void) flb_destroy(ctx); } +void flb_test_enable_flb_null(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + char *output = NULL; + char *input = "[0, {\"hello\":null}]"; + char *result; + struct flb_lib_out_cb cb_data; + + char *script_body = "" + "function lua_main(tag, timestamp, record)\n" + " return 1, timestamp, record\n" + "end\n"; + + clear_output(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + ret = create_script(script_body, strlen(script_body)); + TEST_CHECK(ret == 0); + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "lua", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "call", "lua_main", + "script", TMP_LUA_PATH, + "enable_flb_null", "true", + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + flb_lib_push(ctx, in_ffd, input, strlen(input)); + wait_with_timeout(2000, &output); + result = strstr(output, "\"hello\":null"); + if(!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + + /* clean up */ + flb_lib_free(output); + delete_script(); + + flb_stop(ctx); + flb_destroy(ctx); +} + /* https://github.com/fluent/fluent-bit/issues/5496 */ void flb_test_split_record(void) { @@ -761,6 +831,7 @@ TEST_LIST = { {"type_array_key", flb_test_type_array_key}, {"array_contains_null", flb_test_array_contains_null}, {"drop_all_records", flb_test_drop_all_records}, + {"enable_flb_null", flb_test_enable_flb_null}, {"split_record", flb_test_split_record}, {NULL, NULL} }; From 421ca086bfb736933bdaa8530450ea5141735339 Mon Sep 17 00:00:00 2001 From: Takahiro Yamashita Date: Sat, 9 Sep 2023 07:40:19 +0900 Subject: [PATCH 3/3] filter_lua: support enable_flb_null Signed-off-by: Takahiro Yamashita --- plugins/filter_lua/lua.c | 11 +++++++++++ plugins/filter_lua/lua_config.h | 1 + 2 files changed, 12 insertions(+) diff --git a/plugins/filter_lua/lua.c b/plugins/filter_lua/lua.c index a95c87da1fe..bb7bb566ab4 100644 --- a/plugins/filter_lua/lua.c +++ b/plugins/filter_lua/lua.c @@ -61,6 +61,10 @@ static int cb_lua_init(struct flb_filter_instance *f_ins, } ctx->lua = lj; + if (ctx->enable_flb_null) { + flb_lua_enable_flb_null(lj->state); + } + /* Lua script source code */ if (ctx->code) { ret = flb_luajit_load_buffer(ctx->lua, @@ -683,6 +687,13 @@ static struct flb_config_map config_map[] = { "If enabled, Fluent-bit will pass the timestamp as a Lua table " "with keys \"sec\" for seconds since epoch and \"nsec\" for nanoseconds." }, + { + FLB_CONFIG_MAP_BOOL, "enable_flb_null", "false", + 0, FLB_TRUE, offsetof(struct lua_filter, enable_flb_null), + "If enabled, null will be converted to flb_null in Lua. " + "It is useful to prevent removing key/value " + "since nil is a special value to remove key value from map in Lua." + }, {0} }; diff --git a/plugins/filter_lua/lua_config.h b/plugins/filter_lua/lua_config.h index e5cc1a9b4e5..af8d6f12858 100644 --- a/plugins/filter_lua/lua_config.h +++ b/plugins/filter_lua/lua_config.h @@ -35,6 +35,7 @@ struct lua_filter { flb_sds_t buffer; /* json dec buffer */ int protected_mode; /* exec lua function in protected mode */ int time_as_table; /* timestamp as a Lua table */ + int enable_flb_null; /* Use flb_null in Lua */ struct flb_lua_l2c_config l2cc; /* lua -> C config */ struct flb_luajit *lua; /* state context */ struct flb_filter_instance *ins; /* filter instance */