From 5b4b57025111fd3348634f833b9d6456f84fd9a1 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Mon, 9 Dec 2024 17:03:05 +0800 Subject: [PATCH 01/18] feat(duplication): add a new command to shell to list duplications for one or multiple tables --- src/shell/command_helper.h | 25 ++++++-- src/shell/command_utils.h | 33 ++++++---- src/shell/commands.h | 2 + src/shell/commands/detect_hotkey.cpp | 11 +++- src/shell/commands/duplication.cpp | 92 +++++++++++++++++++++++++++- src/shell/main.cpp | 2 + src/utils/strings.h | 12 ++-- 7 files changed, 150 insertions(+), 27 deletions(-) diff --git a/src/shell/command_helper.h b/src/shell/command_helper.h index db8e3c6f3b..715742cceb 100644 --- a/src/shell/command_helper.h +++ b/src/shell/command_helper.h @@ -980,15 +980,15 @@ class aggregate_stats_calcs const auto param = cmd(param_index++).str(); \ ::dsn::utils::split_args(param.c_str(), container, ','); \ if (container.empty()) { \ - fmt::print(stderr, \ + SHELL_PRINTLN_ERROR( \ "invalid command, '{}' should be in the form of 'val1,val2,val3' and " \ - "should not be empty\n", \ + "should not be empty", \ param); \ return false; \ } \ std::set str_set(container.begin(), container.end()); \ if (str_set.size() != container.size()) { \ - fmt::print(stderr, "invalid command, '{}' has duplicate values\n", param); \ + SHELL_PRINTLN_ERROR("invalid command, '{}' has duplicate values", param); \ return false; \ } \ } while (false) @@ -1005,7 +1005,7 @@ class aggregate_stats_calcs do { \ const auto param = cmd(param_index++).str(); \ if (!::dsn::buf2uint32(param, value)) { \ - fmt::print(stderr, "invalid command, '{}' should be an unsigned integer\n", param); \ + SHELL_PRINTLN_ERROR("invalid command, '{}' should be an unsigned integer", param); \ return false; \ } \ } while (false) @@ -1019,7 +1019,7 @@ class aggregate_stats_calcs do { \ const auto param = cmd(__VA_ARGS__, (def_val)).str(); \ if (!::dsn::buf2uint32(param, value)) { \ - fmt::print(stderr, "invalid command, '{}' should be an unsigned integer\n", param); \ + SHELL_PRINTLN_ERROR("invalid command, '{}' should be an unsigned integer", param); \ return false; \ } \ } while (false) @@ -1034,13 +1034,26 @@ class aggregate_stats_calcs for (const auto &str : strs) { \ uint32_t v; \ if (!::dsn::buf2uint32(str, v)) { \ - fmt::print(stderr, "invalid command, '{}' should be an unsigned integer\n", str); \ + SHELL_PRINTLN_ERROR("invalid command, '{}' should be an unsigned integer", str); \ return false; \ } \ container.insert(v); \ } \ } while (false) +#define PARSE_OPT_ENUM(enum_val, invalid_val, ...) \ + do { \ + const std::string __str(cmd(__VA_ARGS__, {}).str());\ + if (!__str.empty()) { \ + const auto &__val= enum_from_string(__str.c_str(), invalid_val);\ + if (__val==invalid_val) {\ + SHELL_PRINTLN_ERROR("invalid enum {}", __str);\ + return false; \ + } \ + enum_val = __val;\ + } \ + } while (false) + #define RETURN_FALSE_IF_NOT(expr, ...) \ do { \ if (dsn_unlikely(!(expr))) { \ diff --git a/src/shell/command_utils.h b/src/shell/command_utils.h index 2f254a8c9d..ab180db25f 100644 --- a/src/shell/command_utils.h +++ b/src/shell/command_utils.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -27,6 +28,7 @@ #include #include "shell/argh.h" +#include "utils/errors.h" #include "utils/ports.h" #include "utils/strings.h" @@ -36,35 +38,42 @@ class host_port; struct shell_context; -inline bool validate_cmd(const argh::parser &cmd, +inline dsn::error_s empty_pos_args(const argh::parser &cmd) +{ + if (cmd.size() > 0) { + return FMT_ERR(dsn::ERR_INVALID_PARAMETERS, "there shouldn't be any positional arguments"); + } + + return dsn::error_s::ok(); +} + +inline dsn::error_s validate_cmd(const argh::parser &cmd, const std::set ¶ms, - const std::set &flags) + const std::set &flags, + std::function pos_args_checker) { - if (cmd.size() > 1) { - fmt::print(stderr, "too many params!\n"); - return false; + const auto &result = pos_args_checker(cmd); + if (!result) { + return result; } for (const auto ¶m : cmd.params()) { if (params.find(param.first) == params.end()) { - fmt::print(stderr, "unknown param {} = {}\n", param.first, param.second); - return false; + return FMT_ERR(dsn::ERR_INVALID_PARAMETERS, "unknown param {} = {}", param.first, param.second); } } for (const auto &flag : cmd.flags()) { if (params.find(flag) != params.end()) { - fmt::print(stderr, "missing value of {}\n", flag); - return false; + return FMT_ERR(dsn::ERR_INVALID_PARAMETERS, "missing value of {}", flag); } if (flags.find(flag) == flags.end()) { - fmt::print(stderr, "unknown flag {}\n", flag); - return false; + return FMT_ERR(dsn:ERR_INVALID_PARAMETERS, "unknown flag\n", flag); } } - return true; + return dsn::error_s::ok(); } bool validate_ip(shell_context *sc, diff --git a/src/shell/commands.h b/src/shell/commands.h index 2e6044b064..6bc991dcc2 100644 --- a/src/shell/commands.h +++ b/src/shell/commands.h @@ -260,6 +260,8 @@ bool add_dup(command_executor *e, shell_context *sc, arguments args); bool query_dup(command_executor *e, shell_context *sc, arguments args); +bool ls_dups(command_executor *e, shell_context *sc, arguments args); + bool remove_dup(command_executor *e, shell_context *sc, arguments args); bool start_dup(command_executor *e, shell_context *sc, arguments args); diff --git a/src/shell/commands/detect_hotkey.cpp b/src/shell/commands/detect_hotkey.cpp index b93acd1d05..85b0784a52 100644 --- a/src/shell/commands/detect_hotkey.cpp +++ b/src/shell/commands/detect_hotkey.cpp @@ -72,7 +72,7 @@ bool detect_hotkey(command_executor *e, shell_context *sc, arguments args) // detect_hotkey // <-a|--app_id str><-p|--partition_index num><-t|--hotkey_type read|write> // <-c|--detect_action start|stop|query><-d|--address str> - const std::set params = {"a", + static const std::set params = {"a", "app_id", "p", "partition_index", @@ -82,9 +82,14 @@ bool detect_hotkey(command_executor *e, shell_context *sc, arguments args) "hotkey_type", "d", "address"}; - const std::set flags = {}; + static const std::set flags = {}; + argh::parser cmd(args.argc, args.argv, argh::parser::PREFER_PARAM_FOR_UNREG_OPTION); - if (!validate_cmd(cmd, params, flags)) { + + const auto &check = validate_cmd(cmd, params, flags, empty_pos_args); + if (!check) { + // TODO(wangdan): use SHELL_PRINT* macros instead. + fmt::print(stderr, "{}\n", check.description()); return false; } diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index 82237af1a7..fb00e8c7fb 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -221,7 +221,7 @@ bool query_dup(command_executor *e, shell_context *sc, arguments args) const auto &resp = err_resp.get_value(); fmt::println("duplications of app [{}] are listed as below:", app_name); - dsn::utils::table_printer printer; + dsn::utils::table_printer printer(); printer.add_title("dup_id"); printer.add_column("status"); printer.add_column("remote cluster"); @@ -243,6 +243,96 @@ bool query_dup(command_executor *e, shell_context *sc, arguments args) return true; } +namespace { + +struct list_dups_options +{ + bool without_base{false}; + bool list_partitions{false}; + bool check_progress{false}; + uint32_t progress_gap{0}; + bool show_unfinishd{false}; +}; + +void print_dups(const list_dups_options &options, const std::map &app_states) +{ + dsn::utils::table_printer printer("duplications"); + printer.add_title("app_name"); + printer.add_column("dup_id", tp_alignment::kRight); + printer.add_column("create_time", tp_alignment::kRight); + printer.add_column("status", tp_alignment::kRight); + printer.add_column("fail_mode", tp_alignment::kRight); + printer.add_column("remote_cluster", tp_alignment::kRight); + printer.add_column("remote_app_name", tp_alignment::kRight); + + for (auto app : app_states) { + std::string create_time; + dsn::utils::time_ms_to_string(info.create_ts, create_time); + + printer.add_row(info.dupid); + printer.append_data(duplication_status_to_string(info.status)); + printer.append_data(info.remote); + printer.append_data(create_time); + + printer.output(std::cout); + std::cout << std::endl; + } + +} + +} // anonymous namespace + +bool ls_dups(command_executor *e, shell_context *sc, arguments args) +{ + // dups [-a|--app_name_pattern str] [-m|--match_type str] [-i|--without_base] + // [-p|--list_partitions] [-c|--check_progress] [-g|--progress_gap num] + // [-u|--show_unfinishd] + + static const std::set params = {"a", + "app_name_pattern", + "m", + "match_type", + "g", + "progress_gap"}; + static const std::set flags = {"i", "without_base", + "p", "list_partitions", "c", "check_progress", + "u", "show_unfinishd"}; + + argh::parser cmd(args.argc, args.argv, argh::parser::PREFER_PARAM_FOR_UNREG_OPTION); + + const auto &check = validate_cmd(cmd, params, flags, empty_pos_args); + if (!check) { + SHELL_PRINTLN_ERROR("{}", check.description()); + return false; + } + + const std::string app_name_pattern(cmd({"-a", "--app_name_pattern"}, {}).str()); + + auto match_type = dsn::utils::pattern_match_type::PMT_MATCH_ALL; + PARSE_OPT_ENUM(match_type, dsn::utils::pattern_match_type::PMT_INVALID, {"-m", "--match_type"}); + + list_dups_options options; + options.without_base = cmd[{"-p", "--list_partitions"}] + options.list_partitions = cmd[{"-p", "--list_partitions"}]; + options.check_progress = cmd[{"-c", "--check_progress"}]; + PARSE_OPT_UINT(options.progress_gap, 0, {"-g", "--progress_gap"}); + options.show_unfinishd = cmd[{"-u", "--show_unfinishd"}]; + + const auto &result= sc->ddl_client->list_dups(app_name_pattern, match_type); + auto status = result.get_error(); + if (status) { + status = FMT_ERR(result.get_value().err, result.get_value().hint_message); + } + + if (!status) { + SHELL_PRINTLN_ERROR("list duplications failed, error={}", status); + return true; + } + + print_dups(options, result.get_value().app_states); + return true; +} + void handle_duplication_modify_response( const std::string &operation, const dsn::error_with &err_resp) { diff --git a/src/shell/main.cpp b/src/shell/main.cpp index 41dd95d144..4852e73ca0 100644 --- a/src/shell/main.cpp +++ b/src/shell/main.cpp @@ -540,6 +540,8 @@ static command_executor commands[] = { "[-r|--remote_replica_count num]", add_dup}, {"query_dup", "query duplication info", " [-d|--detail]", query_dup}, + {"dups", "list duplications of one or multiple tables with specified pattern", + "", ls_dups}, {"remove_dup", "remove duplication", " ", remove_dup}, {"start_dup", "start duplication", " ", start_dup}, {"pause_dup", "pause duplication", " ", pause_dup}, diff --git a/src/utils/strings.h b/src/utils/strings.h index 49ae0b171c..45e581c3f6 100644 --- a/src/utils/strings.h +++ b/src/utils/strings.h @@ -41,11 +41,13 @@ namespace dsn::utils { ENUM_BEGIN2(pattern_match_type::type, pattern_match_type, pattern_match_type::PMT_INVALID) -ENUM_REG(pattern_match_type::PMT_MATCH_EXACT) -ENUM_REG(pattern_match_type::PMT_MATCH_ANYWHERE) -ENUM_REG(pattern_match_type::PMT_MATCH_PREFIX) -ENUM_REG(pattern_match_type::PMT_MATCH_POSTFIX) -ENUM_REG(pattern_match_type::PMT_MATCH_REGEX) + ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_INVALID, invalid) + ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_ALL, all) + ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_EXACT, exact) + ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_ANYWHERE, anywhere) + ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_PREFIX, prefix) + ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_POSTFIX, postfix) + ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_REGEX, regex) ENUM_END2(pattern_match_type::type, pattern_match_type) inline bool is_empty(const char *str) { return str == nullptr || *str == '\0'; } From 17b0dd61f778b117ff31f1c216f28d66decc2798 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Mon, 9 Dec 2024 17:23:32 +0800 Subject: [PATCH 02/18] format --- src/shell/command_helper.h | 34 +++++++++++++-------------- src/shell/command_utils.h | 14 ++++++----- src/shell/commands/detect_hotkey.cpp | 18 +++++++------- src/shell/commands/duplication.cpp | 35 ++++++++++++---------------- src/shell/main.cpp | 3 +-- src/utils/strings.h | 14 +++++------ 6 files changed, 57 insertions(+), 61 deletions(-) diff --git a/src/shell/command_helper.h b/src/shell/command_helper.h index 715742cceb..f459247e12 100644 --- a/src/shell/command_helper.h +++ b/src/shell/command_helper.h @@ -980,15 +980,15 @@ class aggregate_stats_calcs const auto param = cmd(param_index++).str(); \ ::dsn::utils::split_args(param.c_str(), container, ','); \ if (container.empty()) { \ - SHELL_PRINTLN_ERROR( \ - "invalid command, '{}' should be in the form of 'val1,val2,val3' and " \ - "should not be empty", \ - param); \ + SHELL_PRINTLN_ERROR( \ + "invalid command, '{}' should be in the form of 'val1,val2,val3' and " \ + "should not be empty", \ + param); \ return false; \ } \ std::set str_set(container.begin(), container.end()); \ if (str_set.size() != container.size()) { \ - SHELL_PRINTLN_ERROR("invalid command, '{}' has duplicate values", param); \ + SHELL_PRINTLN_ERROR("invalid command, '{}' has duplicate values", param); \ return false; \ } \ } while (false) @@ -1005,7 +1005,7 @@ class aggregate_stats_calcs do { \ const auto param = cmd(param_index++).str(); \ if (!::dsn::buf2uint32(param, value)) { \ - SHELL_PRINTLN_ERROR("invalid command, '{}' should be an unsigned integer", param); \ + SHELL_PRINTLN_ERROR("invalid command, '{}' should be an unsigned integer", param); \ return false; \ } \ } while (false) @@ -1019,7 +1019,7 @@ class aggregate_stats_calcs do { \ const auto param = cmd(__VA_ARGS__, (def_val)).str(); \ if (!::dsn::buf2uint32(param, value)) { \ - SHELL_PRINTLN_ERROR("invalid command, '{}' should be an unsigned integer", param); \ + SHELL_PRINTLN_ERROR("invalid command, '{}' should be an unsigned integer", param); \ return false; \ } \ } while (false) @@ -1034,24 +1034,24 @@ class aggregate_stats_calcs for (const auto &str : strs) { \ uint32_t v; \ if (!::dsn::buf2uint32(str, v)) { \ - SHELL_PRINTLN_ERROR("invalid command, '{}' should be an unsigned integer", str); \ + SHELL_PRINTLN_ERROR("invalid command, '{}' should be an unsigned integer", str); \ return false; \ } \ container.insert(v); \ } \ } while (false) -#define PARSE_OPT_ENUM(enum_val, invalid_val, ...) \ +#define PARSE_OPT_ENUM(enum_val, invalid_val, ...) \ do { \ - const std::string __str(cmd(__VA_ARGS__, {}).str());\ - if (!__str.empty()) { \ - const auto &__val= enum_from_string(__str.c_str(), invalid_val);\ - if (__val==invalid_val) {\ - SHELL_PRINTLN_ERROR("invalid enum {}", __str);\ - return false; \ + const std::string __str(cmd(__VA_ARGS__, "").str()); \ + if (!__str.empty()) { \ + const auto &__val = enum_from_string(__str.c_str(), invalid_val); \ + if (__val == invalid_val) { \ + SHELL_PRINTLN_ERROR("invalid enum {}", __str); \ + return false; \ + } \ + enum_val = __val; \ } \ - enum_val = __val;\ - } \ } while (false) #define RETURN_FALSE_IF_NOT(expr, ...) \ diff --git a/src/shell/command_utils.h b/src/shell/command_utils.h index ab180db25f..eb83108bd2 100644 --- a/src/shell/command_utils.h +++ b/src/shell/command_utils.h @@ -47,10 +47,11 @@ inline dsn::error_s empty_pos_args(const argh::parser &cmd) return dsn::error_s::ok(); } -inline dsn::error_s validate_cmd(const argh::parser &cmd, - const std::set ¶ms, - const std::set &flags, - std::function pos_args_checker) +inline dsn::error_s +validate_cmd(const argh::parser &cmd, + const std::set ¶ms, + const std::set &flags, + std::function pos_args_checker) { const auto &result = pos_args_checker(cmd); if (!result) { @@ -59,7 +60,8 @@ inline dsn::error_s validate_cmd(const argh::parser &cmd, for (const auto ¶m : cmd.params()) { if (params.find(param.first) == params.end()) { - return FMT_ERR(dsn::ERR_INVALID_PARAMETERS, "unknown param {} = {}", param.first, param.second); + return FMT_ERR( + dsn::ERR_INVALID_PARAMETERS, "unknown param {} = {}", param.first, param.second); } } @@ -69,7 +71,7 @@ inline dsn::error_s validate_cmd(const argh::parser &cmd, } if (flags.find(flag) == flags.end()) { - return FMT_ERR(dsn:ERR_INVALID_PARAMETERS, "unknown flag\n", flag); + return FMT_ERR(dsn::ERR_INVALID_PARAMETERS, "unknown flag\n", flag); } } diff --git a/src/shell/commands/detect_hotkey.cpp b/src/shell/commands/detect_hotkey.cpp index 85b0784a52..beef7be39e 100644 --- a/src/shell/commands/detect_hotkey.cpp +++ b/src/shell/commands/detect_hotkey.cpp @@ -73,15 +73,15 @@ bool detect_hotkey(command_executor *e, shell_context *sc, arguments args) // <-a|--app_id str><-p|--partition_index num><-t|--hotkey_type read|write> // <-c|--detect_action start|stop|query><-d|--address str> static const std::set params = {"a", - "app_id", - "p", - "partition_index", - "c", - "hotkey_action", - "t", - "hotkey_type", - "d", - "address"}; + "app_id", + "p", + "partition_index", + "c", + "hotkey_action", + "t", + "hotkey_type", + "d", + "address"}; static const std::set flags = {}; argh::parser cmd(args.argc, args.argv, argh::parser::PREFER_PARAM_FOR_UNREG_OPTION); diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index fb00e8c7fb..d665869614 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -221,7 +221,7 @@ bool query_dup(command_executor *e, shell_context *sc, arguments args) const auto &resp = err_resp.get_value(); fmt::println("duplications of app [{}] are listed as below:", app_name); - dsn::utils::table_printer printer(); + dsn::utils::table_printer printer; printer.add_title("dup_id"); printer.add_column("status"); printer.add_column("remote cluster"); @@ -254,7 +254,8 @@ struct list_dups_options bool show_unfinishd{false}; }; -void print_dups(const list_dups_options &options, const std::map &app_states) +void print_dups(const list_dups_options &options, + const std::map &app_states) { dsn::utils::table_printer printer("duplications"); printer.add_title("app_name"); @@ -266,7 +267,7 @@ void print_dups(const list_dups_options &options, const std::map params = {"a", - "app_name_pattern", - "m", - "match_type", - "g", - "progress_gap"}; - static const std::set flags = {"i", "without_base", - "p", "list_partitions", "c", "check_progress", - "u", "show_unfinishd"}; + static const std::set params = { + "a", "app_name_pattern", "m", "match_type", "g", "progress_gap"}; + static const std::set flags = { + "i", "without_base", "p", "list_partitions", "c", "check_progress", "u", "show_unfinishd"}; argh::parser cmd(args.argc, args.argv, argh::parser::PREFER_PARAM_FOR_UNREG_OPTION); @@ -306,19 +301,19 @@ bool ls_dups(command_executor *e, shell_context *sc, arguments args) return false; } - const std::string app_name_pattern(cmd({"-a", "--app_name_pattern"}, {}).str()); + const std::string app_name_pattern(cmd({"-a", "--app_name_pattern"}, "").str()); auto match_type = dsn::utils::pattern_match_type::PMT_MATCH_ALL; PARSE_OPT_ENUM(match_type, dsn::utils::pattern_match_type::PMT_INVALID, {"-m", "--match_type"}); list_dups_options options; - options.without_base = cmd[{"-p", "--list_partitions"}] + options.without_base = cmd[{"-p", "--list_partitions"}]; options.list_partitions = cmd[{"-p", "--list_partitions"}]; - options.check_progress = cmd[{"-c", "--check_progress"}]; + options.check_progress = cmd[{"-c", "--check_progress"}]; PARSE_OPT_UINT(options.progress_gap, 0, {"-g", "--progress_gap"}); - options.show_unfinishd = cmd[{"-u", "--show_unfinishd"}]; + options.show_unfinishd = cmd[{"-u", "--show_unfinishd"}]; - const auto &result= sc->ddl_client->list_dups(app_name_pattern, match_type); + const auto &result = sc->ddl_client->list_dups(app_name_pattern, match_type); auto status = result.get_error(); if (status) { status = FMT_ERR(result.get_value().err, result.get_value().hint_message); diff --git a/src/shell/main.cpp b/src/shell/main.cpp index 4852e73ca0..d1712c8b7b 100644 --- a/src/shell/main.cpp +++ b/src/shell/main.cpp @@ -540,8 +540,7 @@ static command_executor commands[] = { "[-r|--remote_replica_count num]", add_dup}, {"query_dup", "query duplication info", " [-d|--detail]", query_dup}, - {"dups", "list duplications of one or multiple tables with specified pattern", - "", ls_dups}, + {"dups", "list duplications of one or multiple tables with specified pattern", "", ls_dups}, {"remove_dup", "remove duplication", " ", remove_dup}, {"start_dup", "start duplication", " ", start_dup}, {"pause_dup", "pause duplication", " ", pause_dup}, diff --git a/src/utils/strings.h b/src/utils/strings.h index 45e581c3f6..eb5dcbeba0 100644 --- a/src/utils/strings.h +++ b/src/utils/strings.h @@ -41,13 +41,13 @@ namespace dsn::utils { ENUM_BEGIN2(pattern_match_type::type, pattern_match_type, pattern_match_type::PMT_INVALID) - ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_INVALID, invalid) - ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_ALL, all) - ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_EXACT, exact) - ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_ANYWHERE, anywhere) - ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_PREFIX, prefix) - ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_POSTFIX, postfix) - ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_REGEX, regex) +ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_INVALID, invalid) +ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_ALL, all) +ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_EXACT, exact) +ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_ANYWHERE, anywhere) +ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_PREFIX, prefix) +ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_POSTFIX, postfix) +ENUM_REG_WITH_CUSTOM_NAME(pattern_match_type::PMT_MATCH_REGEX, regex) ENUM_END2(pattern_match_type::type, pattern_match_type) inline bool is_empty(const char *str) { return str == nullptr || *str == '\0'; } From bb9f52a2de8b285c19236ed94556e3440228ce9d Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Mon, 9 Dec 2024 20:35:42 +0800 Subject: [PATCH 03/18] add --- idl/duplication.thrift | 2 + .../duplication/meta_duplication_service.cpp | 1 + src/shell/commands/duplication.cpp | 185 ++++++++++++++---- 3 files changed, 148 insertions(+), 40 deletions(-) diff --git a/idl/duplication.thrift b/idl/duplication.thrift index 3886021fec..90310f710c 100644 --- a/idl/duplication.thrift +++ b/idl/duplication.thrift @@ -187,6 +187,8 @@ struct duplication_app_state // dup id => per-duplication properties 2:map duplications; + + 3:i32 partition_count; } // This request is sent from client to meta. diff --git a/src/meta/duplication/meta_duplication_service.cpp b/src/meta/duplication/meta_duplication_service.cpp index 0d01ee2417..4c5b6495c7 100644 --- a/src/meta/duplication/meta_duplication_service.cpp +++ b/src/meta/duplication/meta_duplication_service.cpp @@ -131,6 +131,7 @@ void meta_duplication_service::list_duplication_info(const duplication_list_requ duplication_app_state dup_app; dup_app.appid = app->app_id; + dup_app.partition_count = app->partition_count; for (const auto &[dup_id, dup] : app->duplications) { dup_app.duplications.emplace(dup_id, dup->to_partition_level_entry_for_list()); diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index d665869614..4799777aa1 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -45,6 +45,150 @@ using dsn::replication::dupid_t; using dsn::replication::duplication_status; +namespace { + +struct list_dups_options +{ + bool without_base{false}; + bool list_partitions{false}; + bool check_progress{false}; + uint32_t progress_gap{0}; + bool show_unfinishd{false}; +}; + +struct list_dups_stat +{ + size_t total_app_count{0}; + size_t duplicating_app_count{0}; + size_t unfinished_app_count{0}; + + size_t total_partition_count{0}; + size_t duplicating_partition_count{0}; + size_t unfinished_partition_count{0}; + + std::map unfinished_apps{}; +}; + +void print_dups(const list_dups_options &options, + const std::map &app_states) +{ + dsn::utils::table_printer printer("duplications"); + printer.add_title("app_name"); + printer.add_column("dup_id", tp_alignment::kRight); + printer.add_column("create_time", tp_alignment::kRight); + printer.add_column("status", tp_alignment::kRight); + printer.add_column("fail_mode", tp_alignment::kRight); + printer.add_column("remote_cluster", tp_alignment::kRight); + printer.add_column("remote_app_name", tp_alignment::kRight); + + for (auto app : app_states) { + /*std::string create_time; + dsn::utils::time_ms_to_string(info.create_ts, create_time); + + printer.add_row(info.dupid); + printer.append_data(duplication_status_to_string(info.status)); + printer.append_data(info.remote); + printer.append_data(create_time); + + printer.output(std::cout); + std::cout << std::endl;*/ + } +} + +void stat_dups(const list_dups_options &options, + const std::map &app_states, + list_dups_stat &stat) +{ + stat.total_app_count = app_states.size(); + + for (const auto &[app_name,app]: app_states) { + stat.total_partition_count += app.partition_count; + if (app.duplications.empty()) { + continue; + } + + ++stat.duplicating_app_count ; + stat.duplicating_partition_count += app.partition_count; + + bool app_unfinished= false; + std::vector unfinished_partitions(app.partition_count); + for (const auto &[dup_id, dup]: dups) { + if (!dup.__isset.partition_states) { + continue; + } + + duplication_app_state unfinished_app; + for(const auto &[partition_id, partition_state]: dup.partition_states) + { + if (partition_state.last_committed_decree < partition_state.confirmed_decree) + { + continue; + } + + if (partition_state.last_committed_decree - partition_state.confirmed_decree + <= options.progress_gap) { + continue; + } + + app_unfinished= true; + if (unfinished_partitions[i] == 0) { + unfinished_partitions[i] = 1; + } + } + } + + for (const auto &val:unfinished_partitions) { + stat.unfinished_partition_count += val; + } + + if (app_unfinished) { + ++stat.unfinished_app_count; + } + } +} + +void check_dups(const list_dups_options &options, + const std::map &app_states) +{ + +} + +void show_dups(const list_dups_options &options, + const std::map &app_states) +{ + list_dups_stat stat; + stat_dups(options, app_states, stat); + + if (options.check_progress) { + check_dup_progress(options, app_states); + return; + } + + dsn::utils::table_printer printer("duplications"); + printer.add_title("app_name"); + printer.add_column("dup_id", tp_alignment::kRight); + printer.add_column("create_time", tp_alignment::kRight); + printer.add_column("status", tp_alignment::kRight); + printer.add_column("fail_mode", tp_alignment::kRight); + printer.add_column("remote_cluster", tp_alignment::kRight); + printer.add_column("remote_app_name", tp_alignment::kRight); + + for (auto app : app_states) { + /*std::string create_time; + dsn::utils::time_ms_to_string(info.create_ts, create_time); + + printer.add_row(info.dupid); + printer.append_data(duplication_status_to_string(info.status)); + printer.append_data(info.remote); + printer.append_data(create_time); + + printer.output(std::cout); + std::cout << std::endl;*/ + } +} + +} // anonymous namespace + bool add_dup(command_executor *e, shell_context *sc, arguments args) { // add_dup [-s|--sst] [-a|--remote_app_name str] @@ -243,45 +387,6 @@ bool query_dup(command_executor *e, shell_context *sc, arguments args) return true; } -namespace { - -struct list_dups_options -{ - bool without_base{false}; - bool list_partitions{false}; - bool check_progress{false}; - uint32_t progress_gap{0}; - bool show_unfinishd{false}; -}; - -void print_dups(const list_dups_options &options, - const std::map &app_states) -{ - dsn::utils::table_printer printer("duplications"); - printer.add_title("app_name"); - printer.add_column("dup_id", tp_alignment::kRight); - printer.add_column("create_time", tp_alignment::kRight); - printer.add_column("status", tp_alignment::kRight); - printer.add_column("fail_mode", tp_alignment::kRight); - printer.add_column("remote_cluster", tp_alignment::kRight); - printer.add_column("remote_app_name", tp_alignment::kRight); - - for (auto app : app_states) { - /*std::string create_time; - dsn::utils::time_ms_to_string(info.create_ts, create_time); - - printer.add_row(info.dupid); - printer.append_data(duplication_status_to_string(info.status)); - printer.append_data(info.remote); - printer.append_data(create_time); - - printer.output(std::cout); - std::cout << std::endl;*/ - } -} - -} // anonymous namespace - bool ls_dups(command_executor *e, shell_context *sc, arguments args) { // dups [-a|--app_name_pattern str] [-m|--match_type str] [-i|--without_base] @@ -324,7 +429,7 @@ bool ls_dups(command_executor *e, shell_context *sc, arguments args) return true; } - print_dups(options, result.get_value().app_states); + show_dups(options, result.get_value().app_states); return true; } From cb9af103bfc18fa4503019dce4c0c444707133a0 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Tue, 10 Dec 2024 00:24:29 +0800 Subject: [PATCH 04/18] add --- src/shell/commands/duplication.cpp | 117 ++++++++++++++++++----------- 1 file changed, 74 insertions(+), 43 deletions(-) diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index 4799777aa1..3901482745 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -56,6 +56,8 @@ struct list_dups_options bool show_unfinishd{false}; }; +using selected_app_dups_map = std::map>>; + struct list_dups_stat { size_t total_app_count{0}; @@ -66,37 +68,12 @@ struct list_dups_stat size_t duplicating_partition_count{0}; size_t unfinished_partition_count{0}; - std::map unfinished_apps{}; + selected_app_dups_map unfinished_apps{}; }; -void print_dups(const list_dups_options &options, - const std::map &app_states) -{ - dsn::utils::table_printer printer("duplications"); - printer.add_title("app_name"); - printer.add_column("dup_id", tp_alignment::kRight); - printer.add_column("create_time", tp_alignment::kRight); - printer.add_column("status", tp_alignment::kRight); - printer.add_column("fail_mode", tp_alignment::kRight); - printer.add_column("remote_cluster", tp_alignment::kRight); - printer.add_column("remote_app_name", tp_alignment::kRight); - - for (auto app : app_states) { - /*std::string create_time; - dsn::utils::time_ms_to_string(info.create_ts, create_time); - - printer.add_row(info.dupid); - printer.append_data(duplication_status_to_string(info.status)); - printer.append_data(info.remote); - printer.append_data(create_time); - - printer.output(std::cout); - std::cout << std::endl;*/ - } -} - -void stat_dups(const list_dups_options &options, +void stat_dups( const std::map &app_states, + uint32_t progress_gap, list_dups_stat &stat) { stat.total_app_count = app_states.size(); @@ -110,14 +87,14 @@ void stat_dups(const list_dups_options &options, ++stat.duplicating_app_count ; stat.duplicating_partition_count += app.partition_count; - bool app_unfinished= false; - std::vector unfinished_partitions(app.partition_count); + size_t unfinished_app_counter= 0; + std::vector unfinished_partition_counters(app.partition_count); + for (const auto &[dup_id, dup]: dups) { if (!dup.__isset.partition_states) { continue; } - duplication_app_state unfinished_app; for(const auto &[partition_id, partition_state]: dup.partition_states) { if (partition_state.last_committed_decree < partition_state.confirmed_decree) @@ -126,25 +103,78 @@ void stat_dups(const list_dups_options &options, } if (partition_state.last_committed_decree - partition_state.confirmed_decree - <= options.progress_gap) { + <= progress_gap) { continue; } - app_unfinished= true; - if (unfinished_partitions[i] == 0) { - unfinished_partitions[i] = 1; + unfinished_app_counter= 1; + if (unfinished_partition_counters[i] == 0) { + unfinished_partition_counters[i] = 1; } + + stat.unfinished_apps[app_name][dup_id].insert(partition_id); } } - for (const auto &val:unfinished_partitions) { - stat.unfinished_partition_count += val; + for (const auto &counter:unfinished_partition_counters) { + stat.unfinished_partition_count += counter; + } + + stat.unfinished_app_count+=unfinished_app_counter; + } +} + +void print_dups( + const std::map &app_states, + bool list_partitions) +{ + dsn::utils::table_printer printer("duplications"); + printer.add_title("app_name"); + printer.add_column("dup_id", tp_alignment::kRight); + printer.add_column("create_time", tp_alignment::kRight); + printer.add_column("status", tp_alignment::kRight); + printer.add_column("remote_cluster", tp_alignment::kRight); + printer.add_column("remote_app_name", tp_alignment::kRight); + + if (list_partitions) + { + printer.add_column("partition_id"); + printer.add_column("confirmed_decree"); + printer.add_column("last_committed_decree"); + } + + for (const auto &[app_name,app]: app_states) { + if (app.duplications.empty()) { + continue; } - if (app_unfinished) { - ++stat.unfinished_app_count; + + for (const auto &[dup_id, dup]: dups) { + if (!list_partitions) { + continue; + } + + if (!dup.__isset.partition_states) { + continue; + } + + printer.add_row(app_name); + printer.append_data(dup_id); + + std::string create_time; + dsn::utils::time_ms_to_string(info.create_ts, create_time); + printer.append_data(create_time); + + printer.append_data(dsn::replication::duplication_status_to_string(dup.status)); + printer.append_data(dup.remote); + printer.append_data(dup.__isset.remote_app_name ? dup.remote_app_name:app_name); + } + } + + printer.output(std::cout); + std::cout << std::endl; } void check_dups(const list_dups_options &options, @@ -153,11 +183,12 @@ void check_dups(const list_dups_options &options, } -void show_dups(const list_dups_options &options, - const std::map &app_states) +void show_dups( + const std::map &app_states, + const list_dups_options &options) { list_dups_stat stat; - stat_dups(options, app_states, stat); + stat_dups(app_states, options.progress_gap, stat); if (options.check_progress) { check_dup_progress(options, app_states); @@ -429,7 +460,7 @@ bool ls_dups(command_executor *e, shell_context *sc, arguments args) return true; } - show_dups(options, result.get_value().app_states); + show_dups(result.get_value().app_states, options); return true; } From 0c8587c79820485ca20abc1985de08b7ebfbf888 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Tue, 10 Dec 2024 10:59:13 +0800 Subject: [PATCH 05/18] format --- src/shell/commands/duplication.cpp | 56 ++++++++++++------------------ 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index 3901482745..3d08e07937 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -71,61 +71,57 @@ struct list_dups_stat selected_app_dups_map unfinished_apps{}; }; -void stat_dups( - const std::map &app_states, - uint32_t progress_gap, - list_dups_stat &stat) +void stat_dups(const std::map &app_states, + uint32_t progress_gap, + list_dups_stat &stat) { stat.total_app_count = app_states.size(); - for (const auto &[app_name,app]: app_states) { + for (const auto &[app_name, app] : app_states) { stat.total_partition_count += app.partition_count; if (app.duplications.empty()) { continue; } - ++stat.duplicating_app_count ; + ++stat.duplicating_app_count; stat.duplicating_partition_count += app.partition_count; - size_t unfinished_app_counter= 0; + size_t unfinished_app_counter = 0; std::vector unfinished_partition_counters(app.partition_count); - for (const auto &[dup_id, dup]: dups) { + for (const auto &[dup_id, dup] : dups) { if (!dup.__isset.partition_states) { continue; } - for(const auto &[partition_id, partition_state]: dup.partition_states) - { - if (partition_state.last_committed_decree < partition_state.confirmed_decree) - { + for (const auto &[partition_id, partition_state] : dup.partition_states) { + if (partition_state.last_committed_decree < partition_state.confirmed_decree) { continue; } - - if (partition_state.last_committed_decree - partition_state.confirmed_decree - <= progress_gap) { + + if (partition_state.last_committed_decree - partition_state.confirmed_decree <= + progress_gap) { continue; } - unfinished_app_counter= 1; + unfinished_app_counter = 1; if (unfinished_partition_counters[i] == 0) { unfinished_partition_counters[i] = 1; } - stat.unfinished_apps[app_name][dup_id].insert(partition_id); + stat.unfinished_apps[app_name][dup_id].insert(partition_id); } } - for (const auto &counter:unfinished_partition_counters) { + for (const auto &counter : unfinished_partition_counters) { stat.unfinished_partition_count += counter; } - stat.unfinished_app_count+=unfinished_app_counter; + stat.unfinished_app_count += unfinished_app_counter; } } -void print_dups( - const std::map &app_states, +void print_dups(const std::map &app_states, bool list_partitions) { dsn::utils::table_printer printer("duplications"); @@ -136,20 +132,18 @@ void print_dups( printer.add_column("remote_cluster", tp_alignment::kRight); printer.add_column("remote_app_name", tp_alignment::kRight); - if (list_partitions) - { + if (list_partitions) { printer.add_column("partition_id"); printer.add_column("confirmed_decree"); printer.add_column("last_committed_decree"); } - for (const auto &[app_name,app]: app_states) { + for (const auto &[app_name, app] : app_states) { if (app.duplications.empty()) { continue; } - - for (const auto &[dup_id, dup]: dups) { + for (const auto &[dup_id, dup] : dups) { if (!list_partitions) { continue; } @@ -167,10 +161,8 @@ void print_dups( printer.append_data(dsn::replication::duplication_status_to_string(dup.status)); printer.append_data(dup.remote); - printer.append_data(dup.__isset.remote_app_name ? dup.remote_app_name:app_name); - + printer.append_data(dup.__isset.remote_app_name ? dup.remote_app_name : app_name); } - } printer.output(std::cout); @@ -180,12 +172,10 @@ void print_dups( void check_dups(const list_dups_options &options, const std::map &app_states) { - } -void show_dups( - const std::map &app_states, - const list_dups_options &options) +void show_dups(const std::map &app_states, + const list_dups_options &options) { list_dups_stat stat; stat_dups(app_states, options.progress_gap, stat); From 198043c83607a359693339e93dc044c06fab541b Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Tue, 10 Dec 2024 11:57:55 +0800 Subject: [PATCH 06/18] add --- src/shell/commands/duplication.cpp | 92 ++++++++++++++++-------------- 1 file changed, 50 insertions(+), 42 deletions(-) diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index 3d08e07937..57e9f530b2 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -121,10 +121,8 @@ void stat_dups(const std::map &app_states, - bool list_partitions) +void add_titles_for_dups(dsn::utils::table_printer &printer, bool list_partitions) { - dsn::utils::table_printer printer("duplications"); printer.add_title("app_name"); printer.add_column("dup_id", tp_alignment::kRight); printer.add_column("create_time", tp_alignment::kRight); @@ -133,10 +131,48 @@ void print_dups(const std::map &app_states, + bool list_partitions) +{ + dsn::utils::table_printer printer("duplications"); + add_titles_for_dups(printer, list_partitions); for (const auto &[app_name, app] : app_states) { if (app.duplications.empty()) { @@ -152,16 +188,7 @@ void print_dups(const std::map &app_states) +void print_selected_dups( + const std::map &app_states, + const selected_app_dups_map &selected_apps) { } @@ -180,31 +208,11 @@ void show_dups(const std::map Date: Tue, 10 Dec 2024 12:40:51 +0800 Subject: [PATCH 07/18] format and fix --- src/shell/commands/duplication.cpp | 51 +++++++++++++++--------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index 57e9f530b2..9acc7634fe 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -49,9 +49,7 @@ namespace { struct list_dups_options { - bool without_base{false}; bool list_partitions{false}; - bool check_progress{false}; uint32_t progress_gap{0}; bool show_unfinishd{false}; }; @@ -89,7 +87,7 @@ void stat_dups(const std::map unfinished_partition_counters(app.partition_count); - for (const auto &[dup_id, dup] : dups) { + for (const auto &[dup_id, dup] : app.duplications) { if (!dup.__isset.partition_states) { continue; } @@ -105,9 +103,9 @@ void stat_dups(const std::map &app_states, - const selected_app_dups_map &selected_apps) + const std::map &app_states, + const selected_app_dups_map &selected_apps) { } @@ -418,14 +419,14 @@ bool query_dup(command_executor *e, shell_context *sc, arguments args) bool ls_dups(command_executor *e, shell_context *sc, arguments args) { - // dups [-a|--app_name_pattern str] [-m|--match_type str] [-i|--without_base] - // [-p|--list_partitions] [-c|--check_progress] [-g|--progress_gap num] + // dups [-a|--app_name_pattern str] [-m|--match_type str] + // [-p|--list_partitions] [-g|--progress_gap num] // [-u|--show_unfinishd] static const std::set params = { "a", "app_name_pattern", "m", "match_type", "g", "progress_gap"}; static const std::set flags = { - "i", "without_base", "p", "list_partitions", "c", "check_progress", "u", "show_unfinishd"}; + "p", "list_partitions", "u", "show_unfinishd"}; argh::parser cmd(args.argc, args.argv, argh::parser::PREFER_PARAM_FOR_UNREG_OPTION); @@ -441,9 +442,7 @@ bool ls_dups(command_executor *e, shell_context *sc, arguments args) PARSE_OPT_ENUM(match_type, dsn::utils::pattern_match_type::PMT_INVALID, {"-m", "--match_type"}); list_dups_options options; - options.without_base = cmd[{"-p", "--list_partitions"}]; options.list_partitions = cmd[{"-p", "--list_partitions"}]; - options.check_progress = cmd[{"-c", "--check_progress"}]; PARSE_OPT_UINT(options.progress_gap, 0, {"-g", "--progress_gap"}); options.show_unfinishd = cmd[{"-u", "--show_unfinishd"}]; From 19899638a46216beecd1f0dc47bf8536fd8a7954 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Tue, 10 Dec 2024 12:41:25 +0800 Subject: [PATCH 08/18] format --- src/shell/commands/duplication.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index 9acc7634fe..4775ad9a37 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -425,8 +425,7 @@ bool ls_dups(command_executor *e, shell_context *sc, arguments args) static const std::set params = { "a", "app_name_pattern", "m", "match_type", "g", "progress_gap"}; - static const std::set flags = { - "p", "list_partitions", "u", "show_unfinishd"}; + static const std::set flags = {"p", "list_partitions", "u", "show_unfinishd"}; argh::parser cmd(args.argc, args.argv, argh::parser::PREFER_PARAM_FOR_UNREG_OPTION); From c6f80b17d78e96a5d58463ebd7f41a90cdd43945 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Tue, 10 Dec 2024 13:55:02 +0800 Subject: [PATCH 09/18] fix clang-tidy and IWYU --- src/shell/command_utils.cpp | 1 + src/shell/command_utils.h | 4 ++-- src/shell/commands/detect_hotkey.cpp | 1 + src/shell/commands/duplication.cpp | 8 +++++++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/shell/command_utils.cpp b/src/shell/command_utils.cpp index c5311cb985..fc2464e88c 100644 --- a/src/shell/command_utils.cpp +++ b/src/shell/command_utils.cpp @@ -17,6 +17,7 @@ #include "command_utils.h" +#include #include #include "client/replication_ddl_client.h" diff --git a/src/shell/command_utils.h b/src/shell/command_utils.h index eb83108bd2..caf71121d7 100644 --- a/src/shell/command_utils.h +++ b/src/shell/command_utils.h @@ -19,15 +19,15 @@ #pragma once -#include +#include #include -#include #include #include #include #include #include "shell/argh.h" +#include "utils/error_code.h" #include "utils/errors.h" #include "utils/ports.h" #include "utils/strings.h" diff --git a/src/shell/commands/detect_hotkey.cpp b/src/shell/commands/detect_hotkey.cpp index beef7be39e..1efa686aa6 100644 --- a/src/shell/commands/detect_hotkey.cpp +++ b/src/shell/commands/detect_hotkey.cpp @@ -30,6 +30,7 @@ #include "shell/command_utils.h" #include "shell/commands.h" #include "utils/error_code.h" +#include "utils/errors.h" #include "utils/string_conv.h" #include "utils/strings.h" diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index 4775ad9a37..da1de0c6fa 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -18,12 +18,16 @@ */ #include -#include +#include +#include #include #include +#include #include #include #include +#include +#include #include #include "client/partition_resolver.h" @@ -33,6 +37,7 @@ #include "shell/argh.h" #include "shell/command_executor.h" #include "shell/command_helper.h" +#include "shell/command_utils.h" #include "shell/commands.h" #include "shell/sds/sds.h" #include "utils/error_code.h" @@ -41,6 +46,7 @@ #include "utils/output_utils.h" #include "utils/string_conv.h" #include "utils/time_utils.h" +#include "utils_types.h" using dsn::replication::dupid_t; using dsn::replication::duplication_status; From 1a202fae7d2771df82852e20e1f73e7ef5f7d1d2 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Tue, 10 Dec 2024 15:26:30 +0800 Subject: [PATCH 10/18] print selected dups --- src/shell/commands/duplication.cpp | 60 +++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index da1de0c6fa..38444e9ae9 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -160,14 +161,19 @@ void add_base_row_for_dups(dsn::utils::table_printer &printer, void add_row_for_dups(dsn::utils::table_printer &printer, bool list_partitions, const std::string &app_name, - const dsn::replication::duplication_entry &dup) + const dsn::replication::duplication_entry &dup, + std::function partition_selector) { - if (list_partitions) { + if (!list_partitions) { add_base_row_for_dups(printer, app_name, dup); return; } for (const auto &[partition_id, partition_state] : dup.partition_states) { + if (partition_selector && !partition_selector(partition_id)) { + continue; + } + add_base_row_for_dups(printer, app_name, dup); printer.append_data(partition_id); printer.append_data(partition_state.confirmed_decree); @@ -175,6 +181,14 @@ void add_row_for_dups(dsn::utils::table_printer &printer, } } +void add_row_for_dups(dsn::utils::table_printer &printer, + bool list_partitions, + const std::string &app_name, + const dsn::replication::duplication_entry &dup) +{ + add_row_for_dups(printer, list_partitions, app_name, dup, std::function()); +} + void print_dups(const std::map &app_states, bool list_partitions) { @@ -205,8 +219,45 @@ void print_dups(const std::map &app_states, - const selected_app_dups_map &selected_apps) + const selected_app_dups_map &selected_apps, + const std::string &topic) { + dsn::utils::table_printer printer(topic); + add_titles_for_dups(printer, true); + + auto selected_app_iter = selected_apps.begin(); + for (const auto &[app_name, app] : app_states) { + if (selected_app_iter == selected_apps.end()) { + break; + } + + if (selected_app_iter->first != app_name) { + continue; + } + + for (const auto &[dup_id, dup] : app.duplications) { + auto selected_dup_iter = selected_app_iter->second.begin(); + if (selected_dup_iter == selected_app_iter->second.end()) { + break; + } + + if (selected_dup_iter->first != dup_id) { + continue; + } + + if (!dup.__isset.partition_states) { + continue; + } + + add_row_for_dups( + printer, true, app_name, dup, [selected_dup_iter](int32_t partition_id) { + return gutil::ContainsKey(selected_dup_iter->second, partition_id); + }); + } + } + + printer.output(std::cout); + std::cout << std::endl; } void show_dups(const std::map &app_states, @@ -218,8 +269,7 @@ void show_dups(const std::map Date: Tue, 10 Dec 2024 17:31:47 +0800 Subject: [PATCH 11/18] print dups --- src/shell/command_utils.h | 3 ++ src/shell/commands/duplication.cpp | 66 ++++++++++++++++++++++++------ src/shell/main.cpp | 7 +++- src/utils/output_utils.h | 29 +++++++++++++ 4 files changed, 91 insertions(+), 14 deletions(-) diff --git a/src/shell/command_utils.h b/src/shell/command_utils.h index caf71121d7..c5c7f32044 100644 --- a/src/shell/command_utils.h +++ b/src/shell/command_utils.h @@ -38,6 +38,8 @@ class host_port; struct shell_context; +// Check if positional arguments are empty, and they should also be empty, which means only +// parameters and flags are needed. inline dsn::error_s empty_pos_args(const argh::parser &cmd) { if (cmd.size() > 0) { @@ -47,6 +49,7 @@ inline dsn::error_s empty_pos_args(const argh::parser &cmd) return dsn::error_s::ok(); } +// Check if the positional arguments are valid, and the parameters and flags are in the given set. inline dsn::error_s validate_cmd(const argh::parser &cmd, const std::set ¶ms, diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index 38444e9ae9..26e7df1c68 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -35,6 +35,7 @@ #include "client/replication_ddl_client.h" #include "common//duplication_common.h" #include "duplication_types.h" +#include "gutil/map_util.h" #include "shell/argh.h" #include "shell/command_executor.h" #include "shell/command_helper.h" @@ -56,9 +57,19 @@ namespace { struct list_dups_options { + // To list partition-level states for a duplication, typically progress info. bool list_partitions{false}; + + // The given max gap between confirmed decree and last committed decree, any gap + // larger than this would be considered as "unfinished". uint32_t progress_gap{0}; + + // Whether partitions with "unfinished" progress should be shown. bool show_unfinishd{false}; + + std::string output_file{}; + + bool json{false}; }; using selected_app_dups_map = std::map>>; @@ -76,6 +87,21 @@ struct list_dups_stat selected_app_dups_map unfinished_apps{}; }; +void attach_dups_stat(const list_dups_stat &stat, dsn::utils::multi_table_printer &multi_printer) +{ + dsn::utils::table_printer printer("summary"); + + printer.add_row_name_and_data("total_app_count", stat.total_app_count); + printer.add_row_name_and_data("duplicating_app_count", stat.duplicating_app_count); + printer.add_row_name_and_data("unfinished_app_count", stat.unfinished_app_count); + + printer.add_row_name_and_data("total_partition_count", stat.total_partition_count); + printer.add_row_name_and_data("duplicating_partition_count", stat.duplicating_partition_count); + printer.add_row_name_and_data("unfinished_partition_count", stat.unfinished_partition_count); + + multi_printer.add(std::move(printer)); +} + void stat_dups(const std::map &app_states, uint32_t progress_gap, list_dups_stat &stat) @@ -189,8 +215,9 @@ void add_row_for_dups(dsn::utils::table_printer &printer, add_row_for_dups(printer, list_partitions, app_name, dup, std::function()); } -void print_dups(const std::map &app_states, - bool list_partitions) +void attach_dups(const std::map &app_states, + bool list_partitions, + dsn::utils::multi_table_printer &multi_printer) { dsn::utils::table_printer printer("duplications"); add_titles_for_dups(printer, list_partitions); @@ -213,14 +240,14 @@ void print_dups(const std::map &app_states, const selected_app_dups_map &selected_apps, - const std::string &topic) + const std::string &topic, + dsn::utils::multi_table_printer &multi_printer) { dsn::utils::table_printer printer(topic); add_titles_for_dups(printer, true); @@ -256,8 +283,7 @@ void print_selected_dups( } } - printer.output(std::cout); - std::cout << std::endl; + multi_printer.add(std::move(printer)); } void show_dups(const std::map &app_states, @@ -266,11 +292,16 @@ void show_dups(const std::map params = { "a", "app_name_pattern", "m", "match_type", "g", "progress_gap"}; static const std::set flags = {"p", "list_partitions", "u", "show_unfinishd"}; argh::parser cmd(args.argc, args.argv, argh::parser::PREFER_PARAM_FOR_UNREG_OPTION); + // Check if input parameters and flags are valid. const auto &check = validate_cmd(cmd, params, flags, empty_pos_args); if (!check) { SHELL_PRINTLN_ERROR("{}", check.description()); return false; } + // Read the parttern of table name with empty string as default. const std::string app_name_pattern(cmd({"-a", "--app_name_pattern"}, "").str()); + // Read the match type of the pattern for table name with "matching all" as default, typically + // requesting all tables owned by this cluster. auto match_type = dsn::utils::pattern_match_type::PMT_MATCH_ALL; PARSE_OPT_ENUM(match_type, dsn::utils::pattern_match_type::PMT_INVALID, {"-m", "--match_type"}); + // Initialize options for listing duplications. list_dups_options options; options.list_partitions = cmd[{"-p", "--list_partitions"}]; PARSE_OPT_UINT(options.progress_gap, 0, {"-g", "--progress_gap"}); options.show_unfinishd = cmd[{"-u", "--show_unfinishd"}]; + options.output_file = cmd({"-o", "--output"}, "").str(); + options.json = cmd[{"-j", "--json"}]; const auto &result = sc->ddl_client->list_dups(app_name_pattern, match_type); auto status = result.get_error(); diff --git a/src/shell/main.cpp b/src/shell/main.cpp index d1712c8b7b..1a7c90d38b 100644 --- a/src/shell/main.cpp +++ b/src/shell/main.cpp @@ -540,7 +540,12 @@ static command_executor commands[] = { "[-r|--remote_replica_count num]", add_dup}, {"query_dup", "query duplication info", " [-d|--detail]", query_dup}, - {"dups", "list duplications of one or multiple tables with specified pattern", "", ls_dups}, + {"dups", + "list duplications of one or multiple tables with specified pattern", + "[-a|--app_name_pattern str] [-m|--match_type str] [-p|--list_partitions] " + "[-g|--progress_gap num] [-u|--show_unfinishd] [-o|--output file_name] " + "[-j|--json]", + ls_dups}, {"remove_dup", "remove duplication", " ", remove_dup}, {"start_dup", "start duplication", " ", start_dup}, {"pause_dup", "pause duplication", " ", pause_dup}, diff --git a/src/utils/output_utils.h b/src/utils/output_utils.h index 201cee938b..02df9441f5 100644 --- a/src/utils/output_utils.h +++ b/src/utils/output_utils.h @@ -22,7 +22,9 @@ #include #include #include // IWYU pragma: keep +#include #include +#include // IWYU pragma: no_include #include // IWYU pragma: keep #include @@ -226,5 +228,32 @@ class multi_table_printer private: std::vector _tps; }; + +template +void output(const std::string &file_path, bool json, const Printer &printer) +{ + std::streambuf *buf = nullptr; + std::ofstream file; + + if (file_path.empty()) { + buf = std::cout.rdbuf(); + } else { + file.open(file_path); + buf = file.rdbuf(); + } + + std::ostream out(buf); + + printer.output(out, + json ? table_printer::output_format::kJsonPretty + : table_printer::output_format::kTabular); +} + +template +void output(bool json, const Printer &printer) +{ + output({}, json, printer); +} + } // namespace utils } // namespace dsn From 3ef652c2e0f42ca44748e2328e6fde7e9215cf0a Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Tue, 10 Dec 2024 18:35:39 +0800 Subject: [PATCH 12/18] stat dups and add comments --- idl/duplication.thrift | 1 + src/client/replication_ddl_client.cpp | 1 + src/shell/command_helper.h | 1 + src/shell/commands/duplication.cpp | 40 +++++++++++++++++++++++++ src/shell/commands/node_management.cpp | 2 ++ src/shell/commands/table_management.cpp | 2 ++ src/utils/output_utils.h | 7 +++-- 7 files changed, 52 insertions(+), 2 deletions(-) diff --git a/idl/duplication.thrift b/idl/duplication.thrift index 90310f710c..dddabf35c7 100644 --- a/idl/duplication.thrift +++ b/idl/duplication.thrift @@ -188,6 +188,7 @@ struct duplication_app_state // dup id => per-duplication properties 2:map duplications; + // The number of partitions for this table. 3:i32 partition_count; } diff --git a/src/client/replication_ddl_client.cpp b/src/client/replication_ddl_client.cpp index e993b35684..99c8f52934 100644 --- a/src/client/replication_ddl_client.cpp +++ b/src/client/replication_ddl_client.cpp @@ -481,6 +481,7 @@ dsn::error_code replication_ddl_client::list_apps(const dsn::app_status::type st } mtp.add(std::move(tp_count)); + // TODO(wangdan): use dsn::utils::output() in output_utils.h instead. mtp.output(out, json ? tp_output_format::kJsonPretty : tp_output_format::kTabular); return dsn::ERR_OK; diff --git a/src/shell/command_helper.h b/src/shell/command_helper.h index f459247e12..d50c60bb7a 100644 --- a/src/shell/command_helper.h +++ b/src/shell/command_helper.h @@ -1041,6 +1041,7 @@ class aggregate_stats_calcs } \ } while (false) +// Parse enum value from the parameters of command line. #define PARSE_OPT_ENUM(enum_val, invalid_val, ...) \ do { \ const std::string __str(cmd(__VA_ARGS__, "").str()); \ diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index 26e7df1c68..187cbbc194 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -67,26 +67,54 @@ struct list_dups_options // Whether partitions with "unfinished" progress should be shown. bool show_unfinishd{false}; + // Specify a file path to output listed duplication info. Empty value means stdout. std::string output_file{}; + // Whether output as json format. bool json{false}; }; using selected_app_dups_map = std::map>>; +using dup_status_stat_map = std::map; +using dup_remote_cluster_stat_map = std::map; struct list_dups_stat { + // Total number of returned tables with specified table name pattern. size_t total_app_count{0}; + + // The number of returned tables that are duplicating. size_t duplicating_app_count{0}; + + // The number of "unfinished" tables for duplication according to specified + // `progress_gap`. size_t unfinished_app_count{0}; + // The number of listed duplications. + size_t duplication_count{0}; + + // The number of listed duplications for each dup status. + dup_status_stat_map dup_status_stats{}; + + // The number of listed duplications for each remote cluster. + dup_remote_cluster_stat_map dup_remote_cluster_stats{}; + + // Total number of returned partitions with specified table name pattern. size_t total_partition_count{0}; + + // The number of returned partitions that are duplicating. size_t duplicating_partition_count{0}; + + // The number of "unfinished" partitions for duplication according to specified + // `progress_gap`. size_t unfinished_partition_count{0}; + // All partitions that are not "unfinished" according to specified `progress_gap` + // organized as each table. selected_app_dups_map unfinished_apps{}; }; +// Attach to printer the summary stats for listed duplications. void attach_dups_stat(const list_dups_stat &stat, dsn::utils::multi_table_printer &multi_printer) { dsn::utils::table_printer printer("summary"); @@ -95,6 +123,14 @@ void attach_dups_stat(const list_dups_stat &stat, dsn::utils::multi_table_printe printer.add_row_name_and_data("duplicating_app_count", stat.duplicating_app_count); printer.add_row_name_and_data("unfinished_app_count", stat.unfinished_app_count); + printer.add_row_name_and_data("duplication_count", stat.duplication_count); + for (const auto &[status, cnt] : stat.dup_status_stats) { + printer.add_row_name_and_data(fmt::format("{}_count", status), cnt); + } + for (const auto &[remote_cluster, cnt] : stat.dup_remote_cluster_stats) { + printer.add_row_name_and_data(fmt::format("{}_count", remote_cluster), cnt); + } + printer.add_row_name_and_data("total_partition_count", stat.total_partition_count); printer.add_row_name_and_data("duplicating_partition_count", stat.duplicating_partition_count); printer.add_row_name_and_data("unfinished_partition_count", stat.unfinished_partition_count); @@ -121,6 +157,10 @@ void stat_dups(const std::map unfinished_partition_counters(app.partition_count); for (const auto &[dup_id, dup] : app.duplications) { + ++stat.duplication_count; + ++stat.dup_status_stats[dsn::replication::duplication_status_to_string(dup.status)]; + ++stat.dup_remote_cluster_stats[dup.remote]; + if (!dup.__isset.partition_states) { continue; } diff --git a/src/shell/commands/node_management.cpp b/src/shell/commands/node_management.cpp index 3b0a823ee4..8d473252bf 100644 --- a/src/shell/commands/node_management.cpp +++ b/src/shell/commands/node_management.cpp @@ -620,6 +620,7 @@ bool ls_nodes(command_executor *, shell_context *sc, arguments args) std::streambuf *buf; std::ofstream of; + // TODO(wangdan): use dsn::utils::output() in output_utils.h instead. if (!output_file.empty()) { of.open(output_file); buf = of.rdbuf(); @@ -701,6 +702,7 @@ bool ls_nodes(command_executor *, shell_context *sc, arguments args) tp_count.add_row_name_and_data("unalive_node_count", status_by_hp.size() - alive_node_count); mtp.add(std::move(tp_count)); + // TODO(wangdan): use dsn::utils::output() in output_utils.h instead. mtp.output(out, json ? tp_output_format::kJsonPretty : tp_output_format::kTabular); return true; diff --git a/src/shell/commands/table_management.cpp b/src/shell/commands/table_management.cpp index 07dc658217..0ab0cd1f4e 100644 --- a/src/shell/commands/table_management.cpp +++ b/src/shell/commands/table_management.cpp @@ -651,6 +651,8 @@ bool app_stat(command_executor *, shell_context *sc, arguments args) (row.rdb_bf_point_positive_total - row.rdb_bf_point_positive_true) + row.rdb_bf_point_negatives)); } + + // TODO(wangdan): use dsn::utils::output() in output_utils.h instead. tp.output(out, json ? tp_output_format::kJsonPretty : tp_output_format::kTabular); return true; diff --git a/src/utils/output_utils.h b/src/utils/output_utils.h index 02df9441f5..ca90e1ba6f 100644 --- a/src/utils/output_utils.h +++ b/src/utils/output_utils.h @@ -19,9 +19,9 @@ // IWYU pragma: no_include #include -#include #include #include // IWYU pragma: keep +#include #include #include #include @@ -225,10 +225,11 @@ class multi_table_printer out << std::endl; } -private: std::vector _tps; }; +// Used as a general interface for printer to output to a file, typically `table_printer` +// and `multi_table_printer`. template void output(const std::string &file_path, bool json, const Printer &printer) { @@ -249,6 +250,8 @@ void output(const std::string &file_path, bool json, const Printer &printer) : table_printer::output_format::kTabular); } +// Used as a general interface for printer to output to stdout, typically `table_printer` +// and `multi_table_printer`. template void output(bool json, const Printer &printer) { From c1baf85b2f252da86535650b1fb84867cc8a1f4d Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Tue, 10 Dec 2024 19:38:28 +0800 Subject: [PATCH 13/18] add comments and fix clang-tidy and IWYU --- src/shell/commands/duplication.cpp | 24 +++++++++++++++++++++++- src/shell/commands/node_management.cpp | 2 +- src/utils/output_utils.cpp | 2 +- src/utils/output_utils.h | 1 - 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index 187cbbc194..7e1ac03498 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -119,10 +119,12 @@ void attach_dups_stat(const list_dups_stat &stat, dsn::utils::multi_table_printe { dsn::utils::table_printer printer("summary"); + // Add stats for tables. printer.add_row_name_and_data("total_app_count", stat.total_app_count); printer.add_row_name_and_data("duplicating_app_count", stat.duplicating_app_count); printer.add_row_name_and_data("unfinished_app_count", stat.unfinished_app_count); + // Add stats for duplications. printer.add_row_name_and_data("duplication_count", stat.duplication_count); for (const auto &[status, cnt] : stat.dup_status_stats) { printer.add_row_name_and_data(fmt::format("{}_count", status), cnt); @@ -131,6 +133,7 @@ void attach_dups_stat(const list_dups_stat &stat, dsn::utils::multi_table_printe printer.add_row_name_and_data(fmt::format("{}_count", remote_cluster), cnt); } + // Add stats for partitions. printer.add_row_name_and_data("total_partition_count", stat.total_partition_count); printer.add_row_name_and_data("duplicating_partition_count", stat.duplicating_partition_count); printer.add_row_name_and_data("unfinished_partition_count", stat.unfinished_partition_count); @@ -138,56 +141,75 @@ void attach_dups_stat(const list_dups_stat &stat, dsn::utils::multi_table_printe multi_printer.add(std::move(printer)); } +// Stats for listed duplications. void stat_dups(const std::map &app_states, uint32_t progress_gap, list_dups_stat &stat) { + // Record as the number of all listed tables. stat.total_app_count = app_states.size(); for (const auto &[app_name, app] : app_states) { + // Sum up as the total number of all listed partitions. stat.total_partition_count += app.partition_count; + if (app.duplications.empty()) { + // No need to stat other items since there is no duplications for this table. continue; } + // There's at least 1 duplication for this table. Sum up for duplicating tables + // with all partitions of each table marked as duplicating. ++stat.duplicating_app_count; stat.duplicating_partition_count += app.partition_count; + // Use individual variables as counter for "unfinished" tables and partitions in + // case one stat is calculated multiple times. Record 1 as the table and partition + // are "unfinished", while keeping 0 as both are "finished".Initialize all of them + // with 0 to sum up later. size_t unfinished_app_counter = 0; std::vector unfinished_partition_counters(app.partition_count); for (const auto &[dup_id, dup] : app.duplications) { + // Count for all duplication-level stats. ++stat.duplication_count; ++stat.dup_status_stats[dsn::replication::duplication_status_to_string(dup.status)]; ++stat.dup_remote_cluster_stats[dup.remote]; if (!dup.__isset.partition_states) { + // Partition-level states are not set. Only to be compatible with old version + // where there is no this field for duplication entry. continue; } for (const auto &[partition_id, partition_state] : dup.partition_states) { if (partition_state.last_committed_decree < partition_state.confirmed_decree) { + // This is unlikely to happen. continue; } if (partition_state.last_committed_decree - partition_state.confirmed_decree <= progress_gap) { + // This partition is defined as "finished". continue; } + // Just assign with 1 to dedup, in case calculated multiple times. unfinished_app_counter = 1; - CHECK_LT(partition_id, unfinished_partition_counters.size()); unfinished_partition_counters[partition_id] = 1; + // Record the partitions that are still "unfinished". stat.unfinished_apps[app_name][dup_id].insert(partition_id); } } + // Sum up for each "unfinished" partition. for (const auto &counter : unfinished_partition_counters) { stat.unfinished_partition_count += counter; } + // Sum up if table is "unfinished". stat.unfinished_app_count += unfinished_app_counter; } } diff --git a/src/shell/commands/node_management.cpp b/src/shell/commands/node_management.cpp index 8d473252bf..109a8b1ccc 100644 --- a/src/shell/commands/node_management.cpp +++ b/src/shell/commands/node_management.cpp @@ -617,7 +617,7 @@ bool ls_nodes(command_executor *, shell_context *sc, arguments args) } // print configuration_list_nodes_response - std::streambuf *buf; + std::streambuf *buf = nullptr; std::ofstream of; // TODO(wangdan): use dsn::utils::output() in output_utils.h instead. diff --git a/src/utils/output_utils.cpp b/src/utils/output_utils.cpp index dfaa799633..057304f2d8 100644 --- a/src/utils/output_utils.cpp +++ b/src/utils/output_utils.cpp @@ -17,7 +17,7 @@ #include "utils/output_utils.h" -#include +#include // IWYU pragma: no_include #include diff --git a/src/utils/output_utils.h b/src/utils/output_utils.h index ca90e1ba6f..0950f92ddc 100644 --- a/src/utils/output_utils.h +++ b/src/utils/output_utils.h @@ -21,7 +21,6 @@ #include #include #include // IWYU pragma: keep -#include #include #include #include From 67c8b82c6452674be7fe86559d35ec26d3a36a12 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Tue, 10 Dec 2024 20:33:12 +0800 Subject: [PATCH 14/18] add comments and refactor --- src/shell/commands/duplication.cpp | 49 ++++++++++++++++++------------ 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index 7e1ac03498..7163c7ffba 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -214,8 +214,10 @@ void stat_dups(const std::map partition_selector) + bool list_partitions, + std::function partition_selector, + dsn::utils::table_printer &printer) { if (!list_partitions) { - add_base_row_for_dups(printer, app_name, dup); + add_base_row_for_dups(app_name, dup, printer); return; } @@ -262,19 +269,19 @@ void add_row_for_dups(dsn::utils::table_printer &printer, continue; } - add_base_row_for_dups(printer, app_name, dup); + add_base_row_for_dups(app_name, dup, printer); printer.append_data(partition_id); printer.append_data(partition_state.confirmed_decree); printer.append_data(partition_state.last_committed_decree); } } -void add_row_for_dups(dsn::utils::table_printer &printer, +void add_row_for_dups(const std::string &app_name, + const dsn::replication::duplication_entry &dup, bool list_partitions, - const std::string &app_name, - const dsn::replication::duplication_entry &dup) + dsn::utils::table_printer &printer) { - add_row_for_dups(printer, list_partitions, app_name, dup, std::function()); + add_row_for_dups(app_name, dup, list_partitions, std::function(), printer); } void attach_dups(const std::map &app_states, @@ -282,7 +289,7 @@ void attach_dups(const std::mapsecond, partition_id); - }); + }, + printer); } } From 8d1104c6ed0fccb9badb65629366edddf436d6ca Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Wed, 11 Dec 2024 11:06:39 +0800 Subject: [PATCH 15/18] fix --- idl/duplication.thrift | 4 +- .../duplication/meta_duplication_service.cpp | 1 + src/shell/commands/duplication.cpp | 167 ++++++++++++------ 3 files changed, 116 insertions(+), 56 deletions(-) diff --git a/idl/duplication.thrift b/idl/duplication.thrift index dddabf35c7..d2460f2fd6 100644 --- a/idl/duplication.thrift +++ b/idl/duplication.thrift @@ -188,8 +188,10 @@ struct duplication_app_state // dup id => per-duplication properties 2:map duplications; + 3:string app_name; + // The number of partitions for this table. - 3:i32 partition_count; + 4:i32 partition_count; } // This request is sent from client to meta. diff --git a/src/meta/duplication/meta_duplication_service.cpp b/src/meta/duplication/meta_duplication_service.cpp index 4c5b6495c7..9f778181e9 100644 --- a/src/meta/duplication/meta_duplication_service.cpp +++ b/src/meta/duplication/meta_duplication_service.cpp @@ -131,6 +131,7 @@ void meta_duplication_service::list_duplication_info(const duplication_list_requ duplication_app_state dup_app; dup_app.appid = app->app_id; + dup_app.app_name = app_name; dup_app.partition_count = app->partition_count; for (const auto &[dup_id, dup] : app->duplications) { diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index 7163c7ffba..76e611b553 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -74,10 +75,18 @@ struct list_dups_options bool json{false}; }; -using selected_app_dups_map = std::map>>; +// app id => (dup id => partition ids) +using selected_app_dups_map = std::map>>; + +// dup status string => dup count using dup_status_stat_map = std::map; + +// dup remote cluster => dup count using dup_remote_cluster_stat_map = std::map; +// app id => duplication_app_state +using ls_app_dups_map = std::map; + struct list_dups_stat { // Total number of returned tables with specified table name pattern. @@ -142,14 +151,12 @@ void attach_dups_stat(const list_dups_stat &stat, dsn::utils::multi_table_printe } // Stats for listed duplications. -void stat_dups(const std::map &app_states, - uint32_t progress_gap, - list_dups_stat &stat) +void stat_dups(const ls_app_dups_map &app_states, uint32_t progress_gap, list_dups_stat &stat) { // Record as the number of all listed tables. stat.total_app_count = app_states.size(); - for (const auto &[app_name, app] : app_states) { + for (const auto &[app_id, app] : app_states) { // Sum up as the total number of all listed partitions. stat.total_partition_count += app.partition_count; @@ -200,7 +207,7 @@ void stat_dups(const std::map partition_selector, dsn::utils::table_printer &printer) { if (!list_partitions) { - add_base_row_for_dups(app_name, dup, printer); + // Only add table-level and duplication-level columns. + add_base_row_for_dups(app_id, app_name, dup, printer); + return; + } + + if (!dup.__isset.partition_states) { + // Partition-level states are not set. Only to be compatible with old version + // where there is no this field for duplication entry. return; } for (const auto &[partition_id, partition_state] : dup.partition_states) { if (partition_selector && !partition_selector(partition_id)) { + // This partition is excluded according to the selector. continue; } - add_base_row_for_dups(app_name, dup, printer); + // Add table-level and duplication-level columns. + add_base_row_for_dups(app_id, app_name, dup, printer); + + // Add partition-level columns. printer.append_data(partition_id); printer.append_data(partition_state.confirmed_decree); printer.append_data(partition_state.last_committed_decree); } } -void add_row_for_dups(const std::string &app_name, +// All partitions for the duplication would be selected into the printer. +void add_row_for_dups(int32_t app_id, + const std::string &app_name, const dsn::replication::duplication_entry &dup, bool list_partitions, dsn::utils::table_printer &printer) { - add_row_for_dups(app_name, dup, list_partitions, std::function(), printer); + add_row_for_dups( + app_id, app_name, dup, list_partitions, std::function(), printer); } -void attach_dups(const std::map &app_states, +// Attach listed duplications to the printer. +void attach_dups(const ls_app_dups_map &app_states, bool list_partitions, dsn::utils::multi_table_printer &multi_printer) { dsn::utils::table_printer printer("duplications"); add_titles_for_dups(list_partitions, printer); - for (const auto &[app_name, app] : app_states) { + for (const auto &[app_id, app] : app_states) { if (app.duplications.empty()) { + // Skip if there is no duplications for this table. continue; } for (const auto &[_, dup] : app.duplications) { - if (!list_partitions) { - continue; - } - - if (!dup.__isset.partition_states) { - continue; - } - - add_row_for_dups(app_name, dup, list_partitions, printer); + add_row_for_dups(app_id, app.app_name, dup, list_partitions, printer); } } multi_printer.add(std::move(printer)); } -void attach_selected_dups( - const std::map &app_states, - const selected_app_dups_map &selected_apps, - const std::string &topic, - dsn::utils::multi_table_printer &multi_printer) +// Attach selected duplications to the printer. +void attach_selected_dups(const ls_app_dups_map &app_states, + const selected_app_dups_map &selected_apps, + const std::string &topic, + dsn::utils::multi_table_printer &multi_printer) { dsn::utils::table_printer printer(topic); + + // Show partition-level columns. add_titles_for_dups(true, printer); + // Find the intersection between listed and selected tables. + auto listed_app_iter = app_states.begin(); auto selected_app_iter = selected_apps.begin(); - for (const auto &[app_name, app] : app_states) { - if (selected_app_iter == selected_apps.end()) { - break; + while (listed_app_iter != app_states.end() && selected_app_iter != selected_apps.end()) { + if (listed_app_iter->first < selected_app_iter->first) { + ++listed_app_iter; + continue; } - if (selected_app_iter->first != app_name) { + if (listed_app_iter->first > selected_app_iter->first) { + ++selected_app_iter; continue; } - for (const auto &[dup_id, dup] : app.duplications) { - auto selected_dup_iter = selected_app_iter->second.begin(); - if (selected_dup_iter == selected_app_iter->second.end()) { - break; - } - - if (selected_dup_iter->first != dup_id) { + // Find the intersection between listed and selected duplications. + auto listed_dup_iter = listed_app_iter->second.duplications.begin(); + auto selected_dup_iter = selected_app_iter->second.begin(); + while (listed_dup_iter != listed_app_iter->second.duplications.end() && + selected_dup_iter != selected_app_iter->second.end()) { + if (listed_dup_iter->first < selected_dup_iter->first) { + ++listed_dup_iter; continue; } - if (!dup.__isset.partition_states) { + if (listed_dup_iter->first > selected_dup_iter->first) { + ++selected_dup_iter; continue; } add_row_for_dups( - app_name, - dup, + listed_app_iter->first, + listed_app_iter->second.app_name, + listed_dup_iter->second, true, [selected_dup_iter](int32_t partition_id) { return gutil::ContainsKey(selected_dup_iter->second, partition_id); }, printer); + + ++listed_dup_iter; + ++selected_dup_iter; } + + ++listed_app_iter; + ++selected_app_iter; } multi_printer.add(std::move(printer)); } -void show_dups(const std::map &app_states, - const list_dups_options &options) +// Print duplications. +void show_dups(const ls_app_dups_map &app_states, const list_dups_options &options) { + // Calculate stats for duplications. list_dups_stat stat; stat_dups(app_states, options.progress_gap, stat); dsn::utils::multi_table_printer multi_printer; + // Attach listed duplications to printer. attach_dups(app_states, options.list_partitions, multi_printer); + // Attach stats to printer. attach_dups_stat(stat, multi_printer); + if (options.show_unfinishd) { + // Attach unfinished duplications with partition-level info to printer. Use "unfinished" + // as the selector to extract all "unfinished" partitions. attach_selected_dups(app_states, stat.unfinished_apps, "unfinished", multi_printer); } + // Printer output info to target file/stdout. dsn::utils::output(options.output_file, options.json, multi_printer); } @@ -614,18 +658,31 @@ bool ls_dups(command_executor *e, shell_context *sc, arguments args) options.output_file = cmd({"-o", "--output"}, "").str(); options.json = cmd[{"-j", "--json"}]; - const auto &result = sc->ddl_client->list_dups(app_name_pattern, match_type); - auto status = result.get_error(); - if (status) { - status = FMT_ERR(result.get_value().err, result.get_value().hint_message); - } + ls_app_dups_map ls_app_dups; + { + const auto &result = sc->ddl_client->list_dups(app_name_pattern, match_type); + auto status = result.get_error(); + if (status) { + status = FMT_ERR(result.get_value().err, result.get_value().hint_message); + } - if (!status) { - SHELL_PRINTLN_ERROR("list duplications failed, error={}", status); - return true; + if (!status) { + SHELL_PRINTLN_ERROR("list duplications failed, error={}", status); + return true; + } + + // Change the key from app name to id, to list tables in the order of app id. + const auto &app_states = result.get_value().app_states; + std::transform( + app_states.begin(), + app_states.end(), + std::inserter(ls_app_dups, ls_app_dups.end()), + [](const std::pair &app) { + return std::make_pair(app.second.appid, app.second); + }); } - show_dups(result.get_value().app_states, options); + show_dups(ls_app_dups, options); return true; } From 34dc2cc6448a75a23665d12785d3e2b6f9b069d5 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Wed, 11 Dec 2024 12:17:35 +0800 Subject: [PATCH 16/18] fix IWYU --- src/shell/commands/duplication.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index 76e611b553..1f37a6a423 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include From 93df1c94eb142ab74320cc6bd47294de9a0aa7dc Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 11 Dec 2024 22:40:36 +0800 Subject: [PATCH 17/18] Update src/shell/command_helper.h --- src/shell/command_helper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shell/command_helper.h b/src/shell/command_helper.h index d50c60bb7a..219bc3e5e2 100644 --- a/src/shell/command_helper.h +++ b/src/shell/command_helper.h @@ -1048,7 +1048,7 @@ class aggregate_stats_calcs if (!__str.empty()) { \ const auto &__val = enum_from_string(__str.c_str(), invalid_val); \ if (__val == invalid_val) { \ - SHELL_PRINTLN_ERROR("invalid enum {}", __str); \ + SHELL_PRINTLN_ERROR("invalid enum: '{}'", __str); \ return false; \ } \ enum_val = __val; \ From 4aa54b5f7317fbe07b8b2e13ab92ed3e4aff3d00 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Thu, 12 Dec 2024 11:23:11 +0800 Subject: [PATCH 18/18] format --- src/shell/command_helper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shell/command_helper.h b/src/shell/command_helper.h index 219bc3e5e2..6c4cafdd79 100644 --- a/src/shell/command_helper.h +++ b/src/shell/command_helper.h @@ -1048,7 +1048,7 @@ class aggregate_stats_calcs if (!__str.empty()) { \ const auto &__val = enum_from_string(__str.c_str(), invalid_val); \ if (__val == invalid_val) { \ - SHELL_PRINTLN_ERROR("invalid enum: '{}'", __str); \ + SHELL_PRINTLN_ERROR("invalid enum: '{}'", __str); \ return false; \ } \ enum_val = __val; \