diff --git a/src/main/java/io/github/dsheirer/gui/SDRTrunk.java b/src/main/java/io/github/dsheirer/gui/SDRTrunk.java index c35be9f54..7d5504f33 100644 --- a/src/main/java/io/github/dsheirer/gui/SDRTrunk.java +++ b/src/main/java/io/github/dsheirer/gui/SDRTrunk.java @@ -189,10 +189,13 @@ public SDRTrunk() EventLogManager eventLogManager = new EventLogManager(aliasModel, mUserPreferences); mPlaylistManager = new PlaylistManager(mUserPreferences, mTunerManager, aliasModel, eventLogManager, mIconModel); + boolean headless = GraphicsEnvironment.isHeadless(); + mDiagnosticMonitor = new DiagnosticMonitor(mUserPreferences, mPlaylistManager.getChannelProcessingManager(), - mTunerManager); + mTunerManager, headless); + mDiagnosticMonitor.start(); - if(!GraphicsEnvironment.isHeadless()) + if(!headless) { mJavaFxWindowManager = new JavaFxWindowManager(mUserPreferences, mTunerManager, mPlaylistManager); } @@ -420,7 +423,7 @@ private void initGUI() processingStatusReportMenuItem.addActionListener(e -> { try { - Path path = mDiagnosticMonitor.generateProcessingDiagnosticReport(); + Path path = mDiagnosticMonitor.generateProcessingDiagnosticReport("User initiated diagnostic report"); JOptionPane.showMessageDialog(mMainGui, "Report created: " + path.toString(), "Processing Status Report Created", JOptionPane.INFORMATION_MESSAGE); @@ -623,6 +626,7 @@ private void initGUI() private void processShutdown() { mLog.info("Application shutdown started ..."); + mDiagnosticMonitor.stop(); mUserPreferences.getSwingPreference().setLocation(WINDOW_FRAME_IDENTIFIER, mMainGui.getLocation()); mUserPreferences.getSwingPreference().setDimension(WINDOW_FRAME_IDENTIFIER, mMainGui.getSize()); mUserPreferences.getSwingPreference().setMaximized(WINDOW_FRAME_IDENTIFIER, diff --git a/src/main/java/io/github/dsheirer/gui/preference/application/ApplicationPreferenceEditor.java b/src/main/java/io/github/dsheirer/gui/preference/application/ApplicationPreferenceEditor.java index 0a8bac211..65d984a8b 100644 --- a/src/main/java/io/github/dsheirer/gui/preference/application/ApplicationPreferenceEditor.java +++ b/src/main/java/io/github/dsheirer/gui/preference/application/ApplicationPreferenceEditor.java @@ -1,6 +1,6 @@ /* * ***************************************************************************** - * Copyright (C) 2014-2023 Dennis Sheirer + * Copyright (C) 2014-2024 Dennis Sheirer * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -23,10 +23,16 @@ import io.github.dsheirer.preference.application.ApplicationPreference; import javafx.geometry.HPos; import javafx.geometry.Insets; +import javafx.geometry.Orientation; import javafx.scene.control.Label; +import javafx.scene.control.Separator; import javafx.scene.control.Spinner; +import javafx.scene.layout.ColumnConstraints; import javafx.scene.layout.GridPane; import javafx.scene.layout.HBox; +import javafx.scene.layout.Priority; +import javafx.scene.layout.VBox; +import org.controlsfx.control.ToggleSwitch; /** @@ -38,6 +44,7 @@ public class ApplicationPreferenceEditor extends HBox private GridPane mEditorPane; private Label mAutoStartTimeoutLabel; private Spinner mTimeoutSpinner; + private ToggleSwitch mAutomaticDiagnosticMonitoringToggle; /** * Constructs an instance @@ -46,7 +53,14 @@ public class ApplicationPreferenceEditor extends HBox public ApplicationPreferenceEditor(UserPreferences userPreferences) { mApplicationPreference = userPreferences.getApplicationPreference(); - getChildren().add(getEditorPane()); + setMaxWidth(Double.MAX_VALUE); + + VBox vbox = new VBox(); + vbox.setMaxHeight(Double.MAX_VALUE); + vbox.setMaxWidth(Double.MAX_VALUE); + vbox.getChildren().add(getEditorPane()); + HBox.setHgrow(vbox, Priority.ALWAYS); + getChildren().add(vbox); } private GridPane getEditorPane() @@ -55,13 +69,31 @@ private GridPane getEditorPane() { int row = 0; mEditorPane = new GridPane(); + mEditorPane.setMaxWidth(Double.MAX_VALUE); mEditorPane.setVgap(10); - mEditorPane.setHgap(10); + mEditorPane.setHgap(3); mEditorPane.setPadding(new Insets(10, 10, 10, 10)); - GridPane.setHalignment(getAutoStartTimeoutLabel(), HPos.RIGHT); - mEditorPane.add(getAutoStartTimeoutLabel(), 0, row); - mEditorPane.add(getTimeoutSpinner(), 1, row); - mEditorPane.add(new Label("seconds"), 2, row); + + Label monitoringLabel = new Label("Application Health and Diagnostic Monitoring."); + mEditorPane.add(monitoringLabel, 0, row, 2, 1); + GridPane.setHalignment(getAutomaticDiagnosticMonitoringToggle(), HPos.RIGHT); + mEditorPane.add(getAutomaticDiagnosticMonitoringToggle(), 0, ++row); + mEditorPane.add(new Label("Enable Diagnostic Monitoring"), 1, row, 2, 1); + + Separator separator = new Separator(Orientation.HORIZONTAL); + GridPane.setHgrow(separator, Priority.ALWAYS); + mEditorPane.add(separator, 0, ++row, 3, 1); + + mEditorPane.add(getAutoStartTimeoutLabel(), 0, ++row, 2, 1); + GridPane.setHalignment(getTimeoutSpinner(), HPos.RIGHT); + mEditorPane.add(getTimeoutSpinner(), 0, ++row); + mEditorPane.add(new Label("seconds"), 1, row); + + ColumnConstraints c1 = new ColumnConstraints(); + c1.setPercentWidth(30); + ColumnConstraints c2 = new ColumnConstraints(); + c2.setHgrow(Priority.ALWAYS); + mEditorPane.getColumnConstraints().addAll(c1, c2); } return mEditorPane; @@ -91,4 +123,20 @@ private Spinner getTimeoutSpinner() return mTimeoutSpinner; } + + /** + * Toggle switch to enable/disable automatic diagnostic monitoring. + */ + private ToggleSwitch getAutomaticDiagnosticMonitoringToggle() + { + if(mAutomaticDiagnosticMonitoringToggle == null) + { + mAutomaticDiagnosticMonitoringToggle = new ToggleSwitch(); + mAutomaticDiagnosticMonitoringToggle.setSelected(mApplicationPreference.isAutomaticDiagnosticMonitoring()); + mAutomaticDiagnosticMonitoringToggle.selectedProperty().addListener((observable, oldValue, enabled) -> + mApplicationPreference.setAutomaticDiagnosticMonitoring(enabled)); + } + + return mAutomaticDiagnosticMonitoringToggle; + } } diff --git a/src/main/java/io/github/dsheirer/monitor/DiagnosticMonitor.java b/src/main/java/io/github/dsheirer/monitor/DiagnosticMonitor.java index 045747298..aaccad210 100644 --- a/src/main/java/io/github/dsheirer/monitor/DiagnosticMonitor.java +++ b/src/main/java/io/github/dsheirer/monitor/DiagnosticMonitor.java @@ -20,52 +20,165 @@ package io.github.dsheirer.monitor; import io.github.dsheirer.controller.channel.ChannelProcessingManager; +import io.github.dsheirer.log.LoggingSuppressor; import io.github.dsheirer.preference.UserPreferences; import io.github.dsheirer.source.tuner.manager.TunerManager; +import io.github.dsheirer.util.ThreadPool; import io.github.dsheirer.util.TimeStamp; import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.util.Enumeration; +import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import java.util.jar.Attributes; import java.util.jar.Manifest; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.swing.JOptionPane; + /** * Utility class for monitoring system components and producing logging reports. */ public class DiagnosticMonitor { private static final Logger LOGGER = LoggerFactory.getLogger(DiagnosticMonitor.class); + private final LoggingSuppressor LOG_SUPPRESSOR = new LoggingSuppressor(LOGGER); private static final String DIVIDER = "\n\n=========================================================================\n\n"; private UserPreferences mUserPreferences; private ChannelProcessingManager mChannelProcessingManager; private TunerManager mTunerManager; + private ScheduledFuture mBlockedThreadMonitorHandle; + private BlockedThreadMonitor mMonitor = new BlockedThreadMonitor(); + private boolean mUserAlertedToBlockedThreadCondition = false; + private Map mBlockedThreadDetectionCountMap = new HashMap<>(); + private boolean mHeadless; /** * Constructs an instance * @param userPreferences for application logging directory lookup. + * @param channelProcessingManager for accessing running channel information + * @param tunerManager for accessing allocated tuner channel information + * @param headless to indicate if the thread deadlock monitor should show a user notification. */ public DiagnosticMonitor(UserPreferences userPreferences, ChannelProcessingManager channelProcessingManager, - TunerManager tunerManager) + TunerManager tunerManager, boolean headless) { mUserPreferences = userPreferences; mChannelProcessingManager = channelProcessingManager; mTunerManager = tunerManager; + mHeadless = headless; + } + + /** + * Starts monitoring for blocked threads + */ + public void start() + { + if(mBlockedThreadMonitorHandle != null) + { + mBlockedThreadMonitorHandle.cancel(true); + } + + if(mUserPreferences.getApplicationPreference().isAutomaticDiagnosticMonitoring()) + { + LOGGER.info("Diagnostic monitoring enabled running every 30 seconds"); + mBlockedThreadMonitorHandle = ThreadPool.SCHEDULED.scheduleAtFixedRate(mMonitor, 30, 30, TimeUnit.SECONDS); + } + else + { + LOGGER.info("Diagnostic monitoring disabled per user preference (application)."); + } + } + + /** + * Stops monitoring for blocked threads. + */ + public void stop() + { + if(mBlockedThreadMonitorHandle != null) + { + mBlockedThreadMonitorHandle.cancel(true); + } + + mBlockedThreadMonitorHandle = null; + } + + /** + * Checks for blocked threads and on discovery, generates a diagnostic report and notifies the user (once). + */ + private void checkForBlockedThreads() + { + if(!mUserAlertedToBlockedThreadCondition) + { + try + { + ThreadMXBean bean = ManagementFactory.getThreadMXBean(); + + long ids[] = bean.findDeadlockedThreads(); + + if(ids != null) + { + mUserAlertedToBlockedThreadCondition = true; + + ThreadInfo threadInfo[] = bean.getThreadInfo(ids); + + StringBuilder sb = new StringBuilder(); + sb.append("sdrtrunk detected a critical application error with a threading deadlock, described as follows:\n"); + + for (ThreadInfo threadInfo1 : threadInfo) + { + sb.append("Thread ID[").append(threadInfo1.getThreadId()); + sb.append("] Name [").append(threadInfo1.getThreadName()); + sb.append("] Lock [").append(threadInfo1.getLockName()); + sb.append("] Owned By [ID:").append(threadInfo1.getLockOwnerId()); + sb.append(" | NAME:").append(threadInfo1.getLockName()); + sb.append("]\n"); + } + + LOGGER.error(sb.toString()); + Path reportPath = generateProcessingDiagnosticReport(sb + DIVIDER); + LOGGER.error("Thread deadlock report generated: " + reportPath); + + if(!mHeadless) + { + String title = "sdrtrunk: Critical Error Detected"; + String message = "The sdrtrunk application has detected a thread deadlock situation.\n" + + "The application may degrade over time and eventually run out of memory.\n" + + "A diagnostic report was generated. Please open an issue on the GitHub\n" + + "website and attach this diagnostic report:\n\n" + reportPath.toString(); + JOptionPane.showMessageDialog(null, message, title, JOptionPane.ERROR_MESSAGE); + } + } + } + catch(Exception e) + { + LOG_SUPPRESSOR.error("run error", 1, "Error while monitoring for deadlocked " + + "threads: " + e.getLocalizedMessage()); + //Set the flag so that we don't try to run again. + mUserAlertedToBlockedThreadCondition = true; + } + } } /** * Creates a diagnostic report containing state information for channels that are in a processing state. + * @param message to prepend to the report * @return path for the log file that was created. */ - public Path generateProcessingDiagnosticReport() throws IOException + public Path generateProcessingDiagnosticReport(String message) throws IOException { StringBuilder sb = new StringBuilder(); - sb.append("sdrtrunk Processing Diagnostic Report\n"); + sb.append(message); + sb.append("\n\nsdrtrunk Processing Diagnostic Report\n"); sb.append(DIVIDER); sb.append(getEnvironmentReport()); sb.append(DIVIDER); @@ -190,4 +303,23 @@ public Attributes findManifestAttributes() { return null; } + + /** + * Runnable to periodically check for blocked threads + */ + public class BlockedThreadMonitor implements Runnable + { + @Override + public void run() + { + try + { + checkForBlockedThreads(); + } + catch(Throwable t) + { + LOG_SUPPRESSOR.error("Error", 3, "Error while checking for blocked threads", t); + } + } + } } diff --git a/src/main/java/io/github/dsheirer/preference/application/ApplicationPreference.java b/src/main/java/io/github/dsheirer/preference/application/ApplicationPreference.java index 17c9a8714..58d16914f 100644 --- a/src/main/java/io/github/dsheirer/preference/application/ApplicationPreference.java +++ b/src/main/java/io/github/dsheirer/preference/application/ApplicationPreference.java @@ -1,6 +1,6 @@ /* * ***************************************************************************** - * Copyright (C) 2014-2023 Dennis Sheirer + * Copyright (C) 2014-2024 Dennis Sheirer * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -31,11 +31,13 @@ */ public class ApplicationPreference extends Preference { + private static final String PREFERENCE_KEY_CHANNEL_AUTO_DIAGNOSTIC_MONITORING = "automatic.diagnostic.monitoring"; private static final String PREFERENCE_KEY_CHANNEL_AUTO_START_TIMEOUT = "channel.auto.start.timeout"; private final static Logger mLog = LoggerFactory.getLogger(ApplicationPreference.class); private Preferences mPreferences = Preferences.userNodeForPackage(ApplicationPreference.class); private Integer mChannelAutoStartTimeout; + private Boolean mAutomaticDiagnosticMonitoring; /** * Constructs an instance @@ -77,4 +79,29 @@ public void setChannelAutoStartTimeout(int timeout) mPreferences.putInt(PREFERENCE_KEY_CHANNEL_AUTO_START_TIMEOUT, timeout); notifyPreferenceUpdated(); } + + /** + * Indicates if automatic diagnostic monitoring is enabled. + * @return enabled. + */ + public boolean isAutomaticDiagnosticMonitoring() + { + if(mAutomaticDiagnosticMonitoring == null) + { + mAutomaticDiagnosticMonitoring = mPreferences.getBoolean(PREFERENCE_KEY_CHANNEL_AUTO_DIAGNOSTIC_MONITORING, true); + } + + return mAutomaticDiagnosticMonitoring; + } + + /** + * Sets the enabled state for automatic diagnostic monitoring. + * @param enabled true to turn on monitoring. + */ + public void setAutomaticDiagnosticMonitoring(boolean enabled) + { + mAutomaticDiagnosticMonitoring = enabled; + mPreferences.putBoolean(PREFERENCE_KEY_CHANNEL_AUTO_DIAGNOSTIC_MONITORING, enabled); + notifyPreferenceUpdated(); + } }