Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cleanup(sinsp/metrics): assorted cleanups for robustness #1950

Merged
merged 3 commits into from
Aug 20, 2024
Merged

cleanup(sinsp/metrics): assorted cleanups for robustness #1950

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
725b633
cleanup(sinsp/metrics): remove raw_name as label
incertum Jul 5, 2024
3dbfd82
cleanup(sinsp/metrics): add prometheus metric and label names sanitiz…
incertum Jul 8, 2024
3bd4dc8
cleanup(metrics/prometheus): collapse sanitization method
incertum Jul 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 38 additions & 7 deletions userspace/libsinsp/metrics_collector.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ limitations under the License.
static re2::RE2 s_libs_metrics_units_suffix_pre_prometheus_text_conversion("(_kb|_bytes|_mb|_perc|_percentage|_ratio|_ns|_ts|_sec|_total)", re2::RE2::POSIX);
static re2::RE2 s_libs_metrics_units_memory_suffix("(_kb|_bytes)", re2::RE2::POSIX);
static re2::RE2 s_libs_metrics_units_perc_suffix("(_perc)", re2::RE2::POSIX);
static re2::RE2 s_libs_metrics_banned_prometheus_naming_characters("(\\.)", re2::RE2::POSIX);

// For simplicity, needs to stay in sync w/ typedef enum metrics_v2_value_unit
// https://prometheus.io/docs/practices/naming/ or https://prometheus.io/docs/practices/naming/#base-units.
Expand Down Expand Up @@ -91,6 +90,20 @@ std::string metric_value_to_text(const metrics_v2& metric)
return value_text;
}

std::string prometheus_sanitize_metric_name(const std::string& name, const RE2& invalid_chars = RE2("[^a-zA-Z0-9_:]"))
{
// https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
std::string sanitized_name = name;
RE2::GlobalReplace(&sanitized_name, invalid_chars, "_");
RE2::GlobalReplace(&sanitized_name, "_+", "_");
// Ensure it starts with a letter or underscore (if empty after sanitizing, set to "_")
if (sanitized_name.empty() || (!std::isalpha(sanitized_name.front()) && sanitized_name.front() != '_'))
{
sanitized_name = "_" + sanitized_name;
}
return sanitized_name;
}

std::string prometheus_qualifier(std::string_view prometheus_namespace, std::string_view prometheus_subsystem)
{
std::string qualifier;
Expand All @@ -108,16 +121,35 @@ std::string prometheus_qualifier(std::string_view prometheus_namespace, std::str

std::string prometheus_exposition_text(std::string_view metric_qualified_name, std::string_view metric_name, std::string_view metric_type_name, std::string_view metric_value, const std::map<std::string, std::string>& const_labels)
{
std::string fqn(metric_qualified_name);
std::string fqn = prometheus_sanitize_metric_name(std::string(metric_qualified_name));
std::string prometheus_text = "# HELP " + fqn + " https://falco.org/docs/metrics/\n";
prometheus_text += "# TYPE " + fqn + " " + std::string(metric_type_name) + "\n";
prometheus_text += fqn;
prometheus_text += "{raw_name=\"" + std::string(metric_name) + "\"" ;
for (const auto& [key, value] : const_labels)
if (!const_labels.empty())
{
static const RE2 label_invalid_chars("[^a-zA-Z0-9_]");
prometheus_text += "{";
bool first_label = true;
for (const auto& [key, value] : const_labels)
{
if (key.empty())
{
continue;
}
if (!first_label)
{
prometheus_text += ",";
} else
{
first_label = false;
}
prometheus_text += prometheus_sanitize_metric_name(key, label_invalid_chars) + "=\"" + value + "\"";
}
prometheus_text += "} "; // the white space at the end is important!
} else
{
prometheus_text += "," + key + "=\"" + value + "\"" ;
prometheus_text += " "; // the white space at the end is important!
}
prometheus_text += "} "; // white space at the end important!
prometheus_text += std::string(metric_value);
prometheus_text += "\n";
return prometheus_text;
Expand Down Expand Up @@ -159,7 +191,6 @@ std::string prometheus_metrics_converter::convert_metric_to_text_prometheus(cons
std::string prometheus_metric_name_fully_qualified = prometheus_qualifier(prometheus_namespace, prometheus_subsystem) + std::string(metric.name) + "_";
// Remove native libs unit suffixes if applicable.
RE2::GlobalReplace(&prometheus_metric_name_fully_qualified, s_libs_metrics_units_suffix_pre_prometheus_text_conversion, "");
RE2::GlobalReplace(&prometheus_metric_name_fully_qualified, s_libs_metrics_banned_prometheus_naming_characters, "_");
prometheus_metric_name_fully_qualified += std::string(metrics_unit_name_mappings_prometheus[metric.unit]);
return prometheus_exposition_text(prometheus_metric_name_fully_qualified,
metric.name,
Expand Down
45 changes: 26 additions & 19 deletions userspace/libsinsp/test/sinsp_metrics.ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,42 +62,42 @@ TEST_F(sinsp_with_test_input, sinsp_libs_metrics_collector_prometheus)
prometheus_text = prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "testns", "falco", {{"example_key1", "example1"},{"example_key2", "example2"}});
prometheus_text_substring = R"(# HELP testns_falco_n_missing_container_images_total https://falco.org/docs/metrics/
# TYPE testns_falco_n_missing_container_images_total gauge
testns_falco_n_missing_container_images_total{raw_name="n_missing_container_images",example_key1="example1",example_key2="example2"} 0
testns_falco_n_missing_container_images_total{example_key1="example1",example_key2="example2"} 0
)";
ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text got\n" << prometheus_text;
// Test only one const_labels
prometheus_text = prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "testns", "falco", {{"example_key1", "example1"}});
prometheus_text_substring = R"(# HELP testns_falco_n_missing_container_images_total https://falco.org/docs/metrics/
# TYPE testns_falco_n_missing_container_images_total gauge
testns_falco_n_missing_container_images_total{raw_name="n_missing_container_images",example_key1="example1"} 0
testns_falco_n_missing_container_images_total{example_key1="example1"} 0
)";
ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text got\n" << prometheus_text;
// Test no const_labels
prometheus_text = prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "testns", "falco");
prometheus_text_substring = R"(# HELP testns_falco_n_missing_container_images_total https://falco.org/docs/metrics/
# TYPE testns_falco_n_missing_container_images_total gauge
testns_falco_n_missing_container_images_total{raw_name="n_missing_container_images"} 0
testns_falco_n_missing_container_images_total 0
)";
ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text got\n" << prometheus_text;
// Test no prometheus_subsytem
prometheus_text = prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "testns");
prometheus_text_substring = R"(# HELP testns_n_missing_container_images_total https://falco.org/docs/metrics/
# TYPE testns_n_missing_container_images_total gauge
testns_n_missing_container_images_total{raw_name="n_missing_container_images"} 0
testns_n_missing_container_images_total 0
)";
ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text got\n" << prometheus_text;
// Test no prometheus_namespace
prometheus_text = prometheus_metrics_converter.convert_metric_to_text_prometheus(metric);
prometheus_text_substring = R"(# HELP n_missing_container_images_total https://falco.org/docs/metrics/
# TYPE n_missing_container_images_total gauge
n_missing_container_images_total{raw_name="n_missing_container_images"} 0
n_missing_container_images_total 0
)";
ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text got\n" << prometheus_text;
// Test no prometheus_namespace, but prometheus_subsytem
prometheus_text = prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "", "falco");
prometheus_text_substring = R"(# HELP falco_n_missing_container_images_total https://falco.org/docs/metrics/
# TYPE falco_n_missing_container_images_total gauge
falco_n_missing_container_images_total{raw_name="n_missing_container_images"} 0
falco_n_missing_container_images_total 0
)";
ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text got\n" << prometheus_text;
} else if (strncmp(metric.name, "memory_rss_bytes", strlen(metric.name)) == 0)
Expand All @@ -107,20 +107,27 @@ falco_n_missing_container_images_total{raw_name="n_missing_container_images"} 0
prometheus_text = prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "testns", "falco");
prometheus_text_substring = R"(# HELP testns_falco_memory_rss_bytes https://falco.org/docs/metrics/
# TYPE testns_falco_memory_rss_bytes gauge
testns_falco_memory_rss_bytes{raw_name="memory_rss_bytes"} )";
testns_falco_memory_rss_bytes )";
ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text got\n" << prometheus_text;
}
}

ASSERT_EQ(metrics_names_all_str_post_unit_conversion_pre_prometheus_text_conversion,
"cpu_usage_ratio memory_rss_bytes memory_vsz_bytes memory_pss_bytes container_memory_used_bytes host_cpu_usage_ratio host_memory_used_bytes host_procs_running host_open_fds n_threads n_fds n_noncached_fd_lookups n_cached_fd_lookups n_failed_fd_lookups n_added_fds n_removed_fds n_stored_evts n_store_evts_drops n_retrieved_evts n_retrieve_evts_drops n_noncached_thread_lookups n_cached_thread_lookups n_failed_thread_lookups n_added_threads n_removed_threads n_drops_full_threadtable n_missing_container_images n_containers");

// Test global wrapper base metrics (pseudo metrics)
prometheus_text = prometheus_metrics_converter.convert_metric_to_text_prometheus("kernel_release", "testns", "falco", {{"kernel_release", "6.6.7-200.fc39.x86_64"}});
prometheus_text_substring = R"(# HELP testns_falco_kernel_release_info https://falco.org/docs/metrics/
# TYPE testns_falco_kernel_release_info gauge
testns_falco_kernel_release_info{raw_name="kernel_release",kernel_release="6.6.7-200.fc39.x86_64"} 1
// Test global wrapper base metrics plus test invalid characters sanitization for the metric and label names (pseudo metrics)
prometheus_text = prometheus_metrics_converter.convert_metric_to_text_prometheus("56kernel_release-:!", "", "", {{"0kernel_release__", "6.6.7-200.fc39.x86_64"}, {"", "empty_key_name"}});
prometheus_text_substring = R"(# HELP _56kernel_release_:_info https://falco.org/docs/metrics/
# TYPE _56kernel_release_:_info gauge
_56kernel_release_:_info{_0kernel_release_="6.6.7-200.fc39.x86_64"} 1
)";

prometheus_text = prometheus_metrics_converter.convert_metric_to_text_prometheus("", "", "", {{"0kernel_release__", "6.6.7-200.fc39.x86_64"}, {"", "empty_key_name"}});
prometheus_text_substring = R"(# HELP _info https://falco.org/docs/metrics/
# TYPE _info gauge
_info{_0kernel_release_="6.6.7-200.fc39.x86_64"} 1
)";

std::cerr << prometheus_text;
ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text got\n" << prometheus_text;

Expand Down Expand Up @@ -194,49 +201,49 @@ testns_falco_kernel_release_info{raw_name="kernel_release",kernel_release="6.6.7
{
prometheus_text_substring = R"(# HELP testns_falco_sys_enter_run_cnt_total https://falco.org/docs/metrics/
# TYPE testns_falco_sys_enter_run_cnt_total counter
testns_falco_sys_enter_run_cnt_total{raw_name="sys_enter.run_cnt"} 76435525241
testns_falco_sys_enter_run_cnt_total 76435525241
)";
ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text got\n" << prometheus_text;
} else if (strncmp(metric.name, "sys_enter.run_time_ns", strlen(metric.name)) == 0)
{
prometheus_text_substring = R"(# HELP testns_falco_sys_enter_run_time_nanoseconds_total https://falco.org/docs/metrics/
# TYPE testns_falco_sys_enter_run_time_nanoseconds_total counter
testns_falco_sys_enter_run_time_nanoseconds_total{raw_name="sys_enter.run_time_ns"} 16269369826392
testns_falco_sys_enter_run_time_nanoseconds_total 16269369826392
)";
ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text got\n" << prometheus_text;
} else if (strncmp(metric.name, "sys_enter.avg_time_ns", strlen(metric.name)) == 0)
{
prometheus_text_substring = R"(# HELP testns_falco_sys_enter_avg_time_nanoseconds https://falco.org/docs/metrics/
# TYPE testns_falco_sys_enter_avg_time_nanoseconds gauge
testns_falco_sys_enter_avg_time_nanoseconds{raw_name="sys_enter.avg_time_ns"} 203
testns_falco_sys_enter_avg_time_nanoseconds 203
)";
ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text got\n" << prometheus_text;
} else if (strncmp(metric.name, "n_drops_buffer_total", strlen(metric.name)) == 0 && strlen(metric.name) == 20) // avoid clash with "n_drops" metric name
{
prometheus_text_substring = R"(# HELP testns_falco_n_drops_buffer_total https://falco.org/docs/metrics/
# TYPE testns_falco_n_drops_buffer_total counter
testns_falco_n_drops_buffer_total{raw_name="n_drops_buffer_total"} 5000
testns_falco_n_drops_buffer_total 5000
)";
ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text got\n" << prometheus_text;
} else if (strncmp(metric.name, "duration_sec", strlen(metric.name)) == 0)
{
prometheus_text_substring = R"(# HELP testns_falco_duration_seconds_total https://falco.org/docs/metrics/
# TYPE testns_falco_duration_seconds_total counter
testns_falco_duration_seconds_total{raw_name="duration_sec"} 144
testns_falco_duration_seconds_total 144
)";
ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text got\n" << prometheus_text;
} else if (strncmp(metric.name, "evt_rate_sec", strlen(metric.name)) == 0)
{
prometheus_text_substring = R"(# HELP testns_falco_evt_rate_seconds https://falco.org/docs/metrics/
# TYPE testns_falco_evt_rate_seconds gauge
testns_falco_evt_rate_seconds{raw_name="evt_rate_sec"} 126065.400000
testns_falco_evt_rate_seconds 126065.400000
)";
ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text got\n" << prometheus_text;
} else if (strncmp(metric.name, "host_boot_ts", strlen(metric.name)) == 0)
{
prometheus_text_substring = R"(# HELP testns_falco_host_boot_timestamp_nanoseconds https://falco.org/docs/metrics/
# TYPE testns_falco_host_boot_timestamp_nanoseconds gauge
testns_falco_host_boot_timestamp_nanoseconds{raw_name="host_boot_ts"} 1708753667000000000
testns_falco_host_boot_timestamp_nanoseconds 1708753667000000000
)";
ASSERT_TRUE(prometheus_text.find(prometheus_text_substring) != std::string::npos) << "Substring not found in prometheus_text got\n" << prometheus_text;
}
Expand Down
Loading