add backtrace for debug builds

conanfile.py

@@ -9,7 +9,7 @@
 
 class SISLConan(ConanFile):
     name = "sisl"
-    version = "12.2.1"
+    version = "12.2.2"
 
     homepage = "https://github.com/eBay/sisl"
     description = "Library for fast data structures, utilities"

src/logging/CMakeLists.txt

@@ -9,11 +9,18 @@ if(${breakpad_FOUND})
   list(APPEND LOGGING_DEPS breakpad::breakpad)
 endif()
 
-add_library(sisl_logging)
-target_sources(sisl_logging PRIVATE
+list(APPEND LOGGING_SOURCE_FILES
   logging.cpp
   stacktrace.cpp
-  )
+)
+if (${CMAKE_BUILD_TYPE} STREQUAL Debug)
+  list(APPEND LOGGING_SOURCE_FILES backtrace.cpp)
+endif()
+
+add_library(sisl_logging)
+target_sources(sisl_logging PRIVATE
+  ${LOGGING_SOURCE_FILES}
+)
 target_link_libraries(sisl_logging PUBLIC ${LOGGING_DEPS} -rdynamic)
 
 if (DEFINED ENABLE_TESTING)

src/logging/backtrace.h

@@ -0,0 +1,134 @@
+/**
+ * Copyright (C) 2017-present Jung-Sang Ahn <[email protected]>
+ * All rights reserved.
+ *
+ * https://github.com/greensky00
+ *
+ * Stack Backtrace
+ * Version: 0.3.5
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * ===========================================================
+ *
+ * Enhanced by hkadayam:
+ *  -  While dlsym is available, backtrace does not provide symbol name, fixed it
+ *     by calculating the offset through dlsym.
+ */
+
+#pragma once
+
+// LCOV_EXCL_START
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#include <functional>
+#include <memory>
+#include <mutex>
+
+#if defined(__linux__) || defined(__APPLE__)
+#include <cxxabi.h>
+#include <execinfo.h>
+#endif
+
+#if defined(__linux__) 
+#include <linux/limits.h>
+#endif
+
+#ifdef __APPLE__
+#include <sys/syslimits.h>
+#endif
+
+namespace backtrace_detail {
+constexpr size_t max_backtrace{256};
+constexpr size_t file_name_length{PATH_MAX};
+constexpr size_t symbol_name_length{1024};
+constexpr size_t address_length{16};
+constexpr uint64_t pipe_timeout_ms{15000}; // 15 seconds.  Addr2line can be extremely slow the first time
+} // namespace backtrace_detail
+
+
+[[maybe_unused]] static size_t stack_backtrace_impl(void** const stack_ptr, const size_t stack_ptr_capacity) {
+    return ::backtrace(stack_ptr, static_cast< int >(stack_ptr_capacity));
+}
+
+#if defined(__linux__)
+[[maybe_unused]] extern size_t stack_interpret_linux_file(const void* const* const stack_ptr, FILE* const stack_file,
+                                                          const size_t stack_size, char* const output_buf,
+                                                          const size_t output_buflen, const bool trim_internal);
+#elif defined(__APPLE__)
+[[maybe_unused]] extern size_t stack_interpret_apple(const void* const* const stack_ptr,
+                                                     const char* const* const stack_msg, const size_t stack_size,
+                                                     char* const output_buf, const size_t output_buflen,
+                                                     [[maybe_unused]] const bool trim_internal);
+#else
+[[maybe_unused]] extern size_t stack_interpret_other(const void* const* const stack_ptr,
+                                                     const char* const* const stack_msg, const size_t stack_size,
+                                                     char* const output_buf, const size_t output_buflen,
+                                                     [[maybe_unused]] const bool trim_internal);
+#endif
+
+[[maybe_unused]] static size_t stack_interpret(void* const* const stack_ptr, const size_t stack_size,
+                                               char* const output_buf, const size_t output_buflen,
+                                               const bool trim_internal) {
+#if defined(__linux__)
+    std::unique_ptr< FILE, std::function< void(FILE* const) > > stack_file{std::tmpfile(), [](FILE* const fp) {
+                                                                               if (fp)
+                                                                                   std::fclose(fp);
+                                                                           }};
+    if (!stack_file)
+        return 0;
+
+    ::backtrace_symbols_fd(stack_ptr, static_cast< int >(stack_size), ::fileno(stack_file.get()));
+
+    const size_t len{
+        stack_interpret_linux_file(stack_ptr, stack_file.get(), stack_size, output_buf, output_buflen, trim_internal)};
+#else
+    const std::unique_ptr< char*, std::function< void(char** const) > > stack_msg{
+        ::backtrace_symbols(stack_ptr, static_cast< int >(stack_size)),
+        [](char** const ptr) { if (ptr) std::free(static_cast< void* >(ptr)); }};
+#if defined(__APPLE__)
+    const size_t len{
+        stack_interpret_apple(stack_ptr, stack_msg.get(), stack_size, output_buf, output_buflen, trim_internal)};
+#else
+    const size_t len{
+        stack_interpret_other(stack_ptr, stack_msg.get(), stack_size, output_buf, output_buflen, trim_internal)};
+#endif
+#endif
+    return len;
+}
+
+[[maybe_unused]] static size_t stack_backtrace(char* const output_buf, const size_t output_buflen,
+                                               const bool trim_internal) {
+    // make this static so no memory allocation needed
+    static std::mutex s_lock;
+    static std::array< void*, backtrace_detail::max_backtrace > stack_ptr;
+    {
+        std::lock_guard< std::mutex > lock{s_lock};
+        const size_t stack_size{stack_backtrace_impl(stack_ptr.data(), stack_ptr.size())};
+        return stack_interpret(stack_ptr.data(), stack_size, output_buf, output_buflen, trim_internal);
+    }
+}
+
+// LCOV_EXCL_STOP

src/logging/logging.cpp

@@ -162,7 +162,6 @@ std::filesystem::path get_base_dir() {
     }
 }
 
-
 static std::filesystem::path log_path(std::string const& name) {
     std::filesystem::path p;
     if (0 < SISL_OPTIONS.count("logfile")) {

src/logging/stacktrace.cpp

@@ -32,17 +32,14 @@
 #include <unistd.h>
 #endif
 
-#include <sisl/logging/logging.h>
-#if defined(__linux__)
-#include <breakpad/client/linux/handler/exception_handler.h>
+#ifndef NDEBUG
+#include "stacktrace_debug.h"
+#else
+#include "stacktrace_release.h"
 #endif
 
 namespace sisl {
 namespace logging {
-static bool g_custom_signal_handler_installed{false};
-static size_t g_custom_signal_handlers{0};
-static bool g_crash_handle_all_threads{true};
-static std::mutex g_hdlr_mutex;
 
 typedef struct SignalHandlerData {
     SignalHandlerData(std::string name, const sig_handler_t handler) : name{std::move(name)}, handler{handler} {}
@@ -121,29 +118,9 @@ static const char* exit_reason_name(const SignalType fatal_id) {
     }
 }
 
-#if defined(__linux__)
-static bool dumpCallback(const google_breakpad::MinidumpDescriptor& descriptor, [[maybe_unused]] void*,
-                         bool succeeded) {
-    std::cerr << std::endl << "Minidump path: " << descriptor.path() << std::endl;
-    return succeeded;
-}
-#endif
-
-static void bt_dumper([[maybe_unused]] const SignalType signal_number) {
-#if defined(__linux__)
-    google_breakpad::ExceptionHandler::WriteMinidump(get_base_dir().string(), dumpCallback, nullptr);
-#endif
-}
-
 static void crash_handler(const SignalType signal_number) {
     LOGCRITICAL("\n * ****Received fatal SIGNAL : {}({})\tPID : {}", exit_reason_name(signal_number), signal_number,
                 ::getpid());
-    const auto flush_logs{[]() { // flush all logs
-        spdlog::apply_all([&](std::shared_ptr< spdlog::logger > l) {
-            if (l) l->flush();
-        });
-        std::this_thread::sleep_for(std::chrono::milliseconds{250});
-    }};
 
     // Only one signal will be allowed past this point
     if (exit_in_progress()) {
@@ -155,10 +132,9 @@ static void crash_handler(const SignalType signal_number) {
     } else {
         flush_logs();
     }
-    spdlog::shutdown();
-
-    bt_dumper(signal_number);
 
+    log_stack_trace(g_crash_handle_all_threads, signal_number);
+    spdlog::shutdown();
     exit_with_default_sighandler(signal_number);
 }
 

src/logging/stacktrace_debug.h

@@ -0,0 +1,148 @@
+#pragma once
+
+#include "backtrace.h"
+#include <sisl/logging/logging.h>
+
+namespace sisl {
+namespace logging {
+
+auto& logget_thread_context = LoggerThreadContext::instance();
+auto logger_thread_registry = logget_thread_context.m_logger_thread_registry;
+
+static std::mutex g_mtx_stack_dump_outstanding;
+static size_t g_stack_dump_outstanding{0};
+static std::condition_variable g_stack_dump_cv;
+static std::array< char, max_stacktrace_size() > g_stacktrace_buff;
+static bool g_custom_signal_handler_installed{false};
+static size_t g_custom_signal_handlers{0};
+static bool g_crash_handle_all_threads{true};
+static std::mutex g_hdlr_mutex;
+
+constexpr uint64_t backtrace_timeout_ms{4 * backtrace_detail::pipe_timeout_ms};
+
+static void bt_dumper([[maybe_unused]] const SignalType signal_number) {
+    g_stacktrace_buff.fill(0);
+    stack_backtrace(g_stacktrace_buff.data(), g_stacktrace_buff.size(), true);
+    bool notify{false};
+    {
+        std::unique_lock lock{g_mtx_stack_dump_outstanding};
+        if (g_stack_dump_outstanding > 0) {
+            --g_stack_dump_outstanding;
+            notify = true;
+        }
+    }
+    if (notify) g_stack_dump_cv.notify_all();
+}
+
+static void log_stack_trace_all_threads() {
+    std::unique_lock logger_lock{logger_thread_registry->m_logger_thread_mutex};
+    auto& logger{GetLogger()};
+    auto& critical_logger{GetCriticalLogger()};
+    size_t thread_count{1};
+
+    const auto dump_thread{[&logger, &critical_logger, &thread_count](const bool signal_thread, const auto thread_id) {
+        if (signal_thread) {
+            const auto log_failure{[&logger, &critical_logger, &thread_count, &thread_id](const char* const msg) {
+                if (logger) {
+#ifndef __APPLE__
+                    logger->critical("Thread ID: {}, Thread num: {} - {}\n", thread_id, thread_count, msg);
+#else
+                    logger->critical("Thread num: {} - {}\n", thread_count, msg);
+#endif
+                    logger->flush();
+                }
+                if (critical_logger) {
+#ifndef __APPLE__
+                    critical_logger->critical("Thread ID: {}, Thread num: {} - {}\n", thread_id, thread_count, msg);
+#else
+                    critical_logger->critical("Thread num: {} - {}\n", thread_count, msg);
+#endif
+                    critical_logger->flush();
+                }
+            }};
+
+            {
+                std::unique_lock outstanding_lock{g_mtx_stack_dump_outstanding};
+                assert(g_stack_dump_outstanding == 0);
+                g_stack_dump_outstanding = 1;
+            }
+            if (!send_thread_signal(thread_id, SIGUSR3)) {
+                {
+                    std::unique_lock outstanding_lock{g_mtx_stack_dump_outstanding};
+                    g_stack_dump_outstanding = 0;
+                }
+                log_failure("Invalid/terminated thread");
+                return;
+            }
+
+            {
+                std::unique_lock outstanding_lock{g_mtx_stack_dump_outstanding};
+                const auto result{g_stack_dump_cv.wait_for(outstanding_lock,
+                                                           std::chrono::milliseconds{backtrace_timeout_ms},
+                                                           [] { return (g_stack_dump_outstanding == 0); })};
+                if (!result) {
+                    g_stack_dump_outstanding = 0;
+                    outstanding_lock.unlock();
+                    log_failure("Timeout waiting for stacktrace");
+                    return;
+                }
+            }
+        } else {
+            // dump the thread without recursive signal
+            g_stacktrace_buff.fill(0);
+            stack_backtrace(g_stacktrace_buff.data(), g_stacktrace_buff.size(), true);
+        }
+
+        if (logger) {
+#ifndef __APPLE__
+            logger->critical("Thread ID: {}, Thread num: {}\n{}", thread_id, thread_count, g_stacktrace_buff.data());
+#else
+            logger->critical("Thread num: {}\n{}", thread_count, g_stacktrace_buff.data());
+#endif
+            logger->flush();
+        }
+        if (critical_logger) {
+#ifndef __APPLE__
+            critical_logger->critical("Thread ID: {}, Thread num: {}\n{}", thread_id, thread_count,
+                                      g_stacktrace_buff.data());
+#else
+            critical_logger->critical("Thread num: {}\n{}", thread_count, g_stacktrace_buff.data());
+#endif
+            critical_logger->flush();
+        }
+    }};
+
+    // First dump this thread context
+    dump_thread(false, logger_thread_ctx.m_thread_id);
+    ++thread_count;
+
+    // dump other threads
+    for (auto* const ctx : logger_thread_registry->m_logger_thread_set) {
+        if (ctx == &logger_thread_ctx) { continue; }
+        dump_thread(true, ctx->m_thread_id);
+        ++thread_count;
+    }
+}
+
+static void flush_logs() { // flush all logs
+    spdlog::apply_all([&](std::shared_ptr< spdlog::logger > l) {
+        if (l) l->flush();
+    });
+    std::this_thread::sleep_for(std::chrono::milliseconds{250});
+}
+
+static void log_stack_trace(const bool all_threads, const SignalType) {
+    if (is_crash_handler_installed() && all_threads) {
+        log_stack_trace_all_threads();
+    } else {
+        // make this static so that no memory allocation is necessary
+        static std::array< char, max_stacktrace_size() > buff;
+        buff.fill(0);
+        [[maybe_unused]] const size_t s{stack_backtrace(buff.data(), buff.size(), true)};
+        LOGCRITICAL("\n\n{}", buff.data());
+    }
+    flush_logs();
+}
+
+} // namespace logging
+} // namespace sisl