diff --git a/fancylog/fancylog.py b/fancylog/fancylog.py
index f76420c..2ef48e1 100644
--- a/fancylog/fancylog.py
+++ b/fancylog/fancylog.py
@@ -243,6 +243,7 @@ def initialise_logger(
     """
     if logger_name:
         logger = logging.getLogger(logger_name)
+        logger.handlers = []
     else:
         logger = logging.getLogger()
 
diff --git a/tests/tests/test_general.py b/tests/tests/test_general.py
index c154029..3f8b623 100644
--- a/tests/tests/test_general.py
+++ b/tests/tests/test_general.py
@@ -1,5 +1,7 @@
 import logging
 
+from rich.logging import RichHandler
+
 import fancylog
 
 lateral_separator = "**************"
@@ -38,3 +40,104 @@ def test_logger_name(tmp_path):
     fancylog.start_logging(tmp_path, fancylog, logger_name=logger_name)
 
     assert logger_name in logging.root.manager.loggerDict
+
+
+def test_correct_handlers_are_set(tmp_path):
+    """
+    Test the handlers on the logger are as specified by the
+    `start_logging` call. Note this cannot be tested
+    on the root logger has it holds handlers that
+    were assigned in earlier tests.
+    """
+    logger_name = "hello_world"
+
+    # Test no handlers are assigned when non requested
+    fancylog.start_logging(
+        tmp_path,
+        fancylog,
+        logger_name=logger_name,
+        log_to_console=False,
+        log_to_file=False,
+    )
+
+    logger = logging.getLogger(logger_name)
+
+    assert logger.handlers == []
+
+    # Test RichHandler only is assigned when requested
+    fancylog.start_logging(
+        tmp_path,
+        fancylog,
+        logger_name=logger_name,
+        log_to_console=True,
+        log_to_file=False,
+    )
+
+    assert len(logger.handlers) == 1
+    assert isinstance(logger.handlers[0], RichHandler)
+
+    # Test FileHandler only is assigned when requested
+    fancylog.start_logging(
+        tmp_path,
+        fancylog,
+        logger_name=logger_name,
+        log_to_console=False,
+        log_to_file=True,
+    )
+
+    assert len(logger.handlers) == 1
+    assert isinstance(logger.handlers[0], logging.FileHandler)
+
+    # Test both handlers are assigned when requested
+    fancylog.start_logging(
+        tmp_path,
+        fancylog,
+        logger_name=logger_name,
+        log_to_console=True,
+        log_to_file=True,
+    )
+
+    assert len(logger.handlers) == 2
+    assert isinstance(logger.handlers[0], logging.FileHandler)
+    assert isinstance(logger.handlers[1], RichHandler)
+
+
+def test_handlers_are_refreshed(tmp_path):
+    """
+    When a named logger is requested, the handlers
+    are reset to that handlers assigned on previous
+    calls are not carried over to the most recent call.
+    """
+    logger_name = "hello_world"
+    fancylog.start_logging(
+        tmp_path,
+        fancylog,
+        logger_name=logger_name,
+        log_to_console=False,
+        log_to_file=False,
+    )
+
+    logger = logging.getLogger(logger_name)
+
+    assert logger.handlers == []
+
+    fancylog.start_logging(
+        tmp_path,
+        fancylog,
+        logger_name=logger_name,
+        log_to_console=True,
+        log_to_file=False,
+    )
+
+    assert len(logger.handlers) == 1
+    assert isinstance(logger.handlers[0], RichHandler)
+
+    fancylog.start_logging(
+        tmp_path,
+        fancylog,
+        logger_name=logger_name,
+        log_to_console=False,
+        log_to_file=False,
+    )
+
+    assert logger.handlers == []