Improvements to icecast failure handling (#1401)

- Add mechanism to persist failure reason while continuing to reconnect
- Add 'warning' attribute for failure conditions that may be temporary,
  but still need to be reported
- Add logging for failures
- Prevent possibly temporary server-side conditions from blocking
  reconnect attempts
- Detect icecast 2.4 auth failures
- Prevent generic errors from overriding existing error state

src/main/java/io/github/dsheirer/audio/broadcast/AbstractAudioBroadcaster.java

@@ -34,6 +34,7 @@ public abstract class AbstractAudioBroadcaster<T extends BroadcastConfiguration>
     private Listener<BroadcastEvent> mBroadcastEventListener;
     private T mBroadcastConfiguration;
     protected ObjectProperty<BroadcastState> mBroadcastState = new SimpleObjectProperty<>(BroadcastState.READY);
+    protected ObjectProperty<BroadcastState> mLastBadBroadcastState = new SimpleObjectProperty<>();
     protected int mStreamedAudioCount = 0;
     protected int mErrorAudioCount = 0;
     protected int mAgedOffAudioCount = 0;
@@ -55,6 +56,14 @@ public ObjectProperty<BroadcastState> broadcastStateProperty()
         return mBroadcastState;
     }
 
+    /**
+     * Observable last bad broadcast state property
+     */
+    public ObjectProperty<BroadcastState> lastBadBroadcastStateProperty()
+    {
+        return mLastBadBroadcastState;
+    }
+
     /**
      * Current state of the broadcastAudio connection
      */
@@ -68,10 +77,26 @@ public BroadcastState getBroadcastState()
      */
     public void setBroadcastState(BroadcastState broadcastState)
     {
+        if(broadcastState == BroadcastState.CONNECTED)
+        {
+            mLastBadBroadcastState.setValue(null);
+        }
+        else if(broadcastState.isErrorState() || broadcastState.isWarningState())
+        {
+            mLastBadBroadcastState.setValue(broadcastState);
+        }
         mBroadcastState.setValue(broadcastState);
         broadcast(new BroadcastEvent(this, BroadcastEvent.Event.BROADCASTER_STATE_CHANGE));
     }
 
+    /**
+     * Last bad state of the broadcastAudio connection
+     */
+    public BroadcastState getLastBadBroadcastState()
+    {
+        return mLastBadBroadcastState.get();
+    }
+
     /**
      * Starts the broadcaster
      */

src/main/java/io/github/dsheirer/audio/broadcast/AudioStreamingBroadcaster.java

@@ -211,9 +211,17 @@ public void setBroadcastState(BroadcastState state)
 
             super.setBroadcastState(state);
 
-            if(mBroadcastState.get() != null && mBroadcastState.get().isErrorState())
+            if(mBroadcastState.get() != null)
             {
-                stop();
+                if(mBroadcastState.get().isErrorState())
+                {
+                    mLog.error("[" + getStreamName() + "] status: " + mBroadcastState.get().toString());
+                    stop();
+                }
+                else if(mBroadcastState.get().isWarningState())
+                {
+                    mLog.warn("[" + getStreamName() + "] status: " + mBroadcastState.get().toString());
+                }
             }
 
             if(!connected())

src/main/java/io/github/dsheirer/audio/broadcast/BroadcastState.java

@@ -23,90 +23,92 @@ public enum BroadcastState
     /**
      * General error in configuration that causes the remote server to reject the connection
      */
-    CONFIGURATION_ERROR("Configuration Error", true),
+    CONFIGURATION_ERROR("Configuration Error", true, false),
 
     /**
      * Connected to the broadcastAudio server and capable of streaming
      */
-    CONNECTED("Connected", false),
+    CONNECTED("Connected", false, false),
 
     /**
      * Connection interrupted, attempting to reconnect.
      */
-    CONNECTING("Connecting", false),
+    CONNECTING("Connecting", false, false),
 
     /**
      * Indicates the configuration is disabled
      */
-    DISABLED("Disabled", true),
+    DISABLED("Disabled", true, false),
 
     /**
      * Disconnected from the broadcastAudio server
      */
-    DISCONNECTED("Disconnected", false),
+    DISCONNECTED("Disconnected", false, false),
 
     /**
      * Invalid credentials - user name or password
      */
-    INVALID_CREDENTIALS("Invalid User Name/Password", true),
+    INVALID_CREDENTIALS("Invalid User Name/Password", true, false),
 
     /**
      * Invalid mount point
      */
-    INVALID_MOUNT_POINT("Invalid Mount/Stream ID", true),
+    INVALID_MOUNT_POINT("Invalid Mount/Stream ID", true, false),
 
     /**
      * Invalid configuration settings
      */
-    INVALID_SETTINGS("Invalid Settings", true),
+    INVALID_SETTINGS("Invalid Settings", true, false),
 
     /**
      * Remote server max sources has been exceeded
      */
-    MAX_SOURCES_EXCEEDED("Max Sources Exceeded", true),
+    MAX_SOURCES_EXCEEDED("Max Sources Exceeded", false, true),
 
     /**
      * Specified mount point is already in use
      */
-    MOUNT_POINT_IN_USE("Mount Point In Use", false),
+    MOUNT_POINT_IN_USE("Mount Point In Use", false, true),
 
     /**
      * Network is unavailable or the server address cannot be resolved
      */
-    NETWORK_UNAVAILABLE("Network Unavailable", false),
+    NETWORK_UNAVAILABLE("Network Unavailable", false, false),
 
     /**
      * Server is not known or reachable
      */
-    NO_SERVER("No Server", false),
+    NO_SERVER("No Server", false, false),
 
     /**
      * Initial state with no connection attempted.
      */
-    READY("Ready", false),
+    READY("Ready", false, false),
 
     /**
      * Error while broadcasting stream data.  Temporary error state to allow connection to be reset.
      */
-    TEMPORARY_BROADCAST_ERROR("Temporary Broadcast Error", false),
+    TEMPORARY_BROADCAST_ERROR("Temporary Broadcast Error", false, false),
 
     /**
      * Unsupported audio format
      */
-    UNSUPPORTED_AUDIO_FORMAT("Unsupported Audio Type", true),
+    UNSUPPORTED_AUDIO_FORMAT("Unsupported Audio Type", true, false),
 
     /**
      * Unspecified error
      */
-    ERROR("Error", true);
+    ERROR("Error", true, false);
 
     private String mLabel;
     private boolean mErrorState;
+    private boolean mWarningState;
 
-    BroadcastState(String label, boolean errorState)
+    BroadcastState(String label, boolean errorState, boolean warningState)
     {
         mLabel = label;
         mErrorState = errorState;
+        mWarningState = warningState;
     }
 
     public String toString()
@@ -118,4 +120,9 @@ public boolean isErrorState()
     {
         return mErrorState;
     }
+
+    public boolean isWarningState()
+    {
+        return mWarningState;
+    }
 }

src/main/java/io/github/dsheirer/audio/broadcast/ConfiguredBroadcast.java

@@ -37,6 +37,7 @@ public class ConfiguredBroadcast
     private BroadcastConfiguration mBroadcastConfiguration;
     private AbstractAudioBroadcaster mAudioBroadcaster;
     private ObjectProperty<BroadcastState> mBroadcastState = new SimpleObjectProperty<>();
+    private ObjectProperty<BroadcastState> mLastBadBroadcastState = new SimpleObjectProperty<>();
 
     /**
      * Constructs an instance
@@ -89,18 +90,28 @@ public ObjectProperty<BroadcastState> broadcastStateProperty()
         return mBroadcastState;
     }
 
+    /**
+     * Last bad broadcast state of the configured audio broadcaster (optional)
+     */
+    public ObjectProperty<BroadcastState> lastBadBroadcastStateProperty()
+    {
+        return mLastBadBroadcastState;
+    }
+
     /**
      * Sets the audio broadcaster
      * @param audioBroadcaster to use for this configuration
      */
     public void setAudioBroadcaster(AbstractAudioBroadcaster audioBroadcaster)
     {
         mBroadcastState.unbind();
+        mLastBadBroadcastState.unbind();
         mAudioBroadcaster = audioBroadcaster;
 
         if(audioBroadcaster != null)
         {
             mBroadcastState.bind(mAudioBroadcaster.broadcastStateProperty());
+            mLastBadBroadcastState.bind(mAudioBroadcaster.lastBadBroadcastStateProperty());
         }
         else
         {
@@ -120,6 +131,7 @@ private void updateBroadcastState()
             {
                 mBroadcastState.setValue(BroadcastState.CONFIGURATION_ERROR);
             }
+            mLastBadBroadcastState.setValue(null);
         }
     }
 

src/main/java/io/github/dsheirer/audio/broadcast/icecast/IcecastHTTPAudioBroadcaster.java

@@ -341,15 +341,29 @@ public void exceptionCaught(IoSession session, Throwable throwable) throws Excep
                                     }
                                     else
                                     {
-                                        mLog.error("String [" + getStreamName() + "] - HTTP protocol decoder error - message:\n\n" + message);
+                                        mLog.error("String [" + getStreamName() + "] - HTTP 403 protocol decoder error - message:\n\n" + message);
+                                        setBroadcastState(BroadcastState.DISCONNECTED);
+                                        disconnect();
+                                    }
+                                    break;
+                                case 401: //Unauthorized
+                                    if(message.toString().contains("Authentication Required"))
+                                    {
+                                        mLog.error("Stream [" + getStreamName() + "] - unable to connect - invalid credentials");
+                                        setBroadcastState(BroadcastState.INVALID_CREDENTIALS);
+                                        disconnect();
+                                    }
+                                    else
+                                    {
+                                        mLog.error("String [" + getStreamName() + "] - HTTP 401 protocol decoder error - message:\n\n" + message);
                                         setBroadcastState(BroadcastState.DISCONNECTED);
                                         disconnect();
                                     }
                                     break;
                                 default:
-                                    mLog.error("String [" + getStreamName() + "] - HTTP protocol decoder error - message:\n\n" + message);
-                                    setBroadcastState(BroadcastState.DISCONNECTED);
-                                    disconnect();
+                                        mLog.error("String [" + getStreamName() + "] - HTTP protocol decoder error - message:\n\n" + message);
+                                        setBroadcastState(BroadcastState.DISCONNECTED);
+                                        disconnect();
                             }
                         }
                         else
@@ -401,9 +415,16 @@ public void messageReceived(IoSession session, Object object) throws Exception
                         disconnect();
                         break;
                     default:
-                        setBroadcastState(BroadcastState.ERROR);
-                        disconnect();
+                        /**
+                         * Only allow a generic error to update state if we've not already experienced a more
+                         * specific error. Otherwise, trailing messages will clear the more meaningful error state.
+                         */
+                        if(!getBroadcastState().isErrorState())
+                        {
+                            setBroadcastState(BroadcastState.ERROR);
+                        }
                         mLog.debug("Unspecified error: " + response.toString() + " Class:" + object.getClass());
+                        disconnect();
                         break;
                 }
             }

src/main/java/io/github/dsheirer/audio/broadcast/icecast/IcecastTCPAudioBroadcaster.java

@@ -347,7 +347,15 @@ else if(message.contains("HTTP/1.1 501"))
                     else
                     {
                         mLog.error("Unrecognized server response:" + message);
-                        setBroadcastState(BroadcastState.ERROR);
+
+                        /**
+                         * Only allow a generic error to update state if we've not already experienced a more
+                         * specific error. Otherwise, trailing messages will clear the more meaningful error state.
+                         */
+                        if(!getBroadcastState().isErrorState())
+                        {
+                            setBroadcastState(BroadcastState.ERROR);
+                        }
                     }
                 }
             }

src/main/java/io/github/dsheirer/gui/playlist/streaming/StreamingEditor.java

@@ -440,7 +440,10 @@ protected void updateItem(Boolean item, boolean empty)
             TableColumn stateColumn = new TableColumn("Stream Status");
             stateColumn.setCellValueFactory(new PropertyValueFactory<>("broadcastState"));
 
-            mConfiguredBroadcastTableView.getColumns().addAll(enabledColumn, nameColumn, typeColumn, stateColumn);
+            TableColumn errorColumn = new TableColumn("Last Error");
+            errorColumn.setCellValueFactory(new PropertyValueFactory<>("lastBadBroadcastState"));
+
+            mConfiguredBroadcastTableView.getColumns().addAll(enabledColumn, nameColumn, typeColumn, stateColumn, errorColumn);
 
             mConfiguredBroadcastTableView.getSelectionModel().selectedItemProperty()
                     .addListener((observable, oldValue, newValue) -> setBroadcastConfiguration(newValue));