From f6c70ced8f26b5494434a842a9fbf8cb76b67a52 Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@swin.edu.au>
Date: Thu, 7 Mar 2024 16:22:26 +1100
Subject: [PATCH 01/21] add a third mode for --usdf to load pickles from USDF
 S3 buckets

---
 .../scheduler_dashboard.py                    | 59 +++++++++++++++++--
 1 file changed, 54 insertions(+), 5 deletions(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index 79fc14c1..8ddeb7f8 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -154,12 +154,14 @@ class Scheduler(param.Parameterized):
         default="",
         label="Scheduler pickle file",
         doc=scheduler_fname_doc,
+        precedence=2,
     )
     widget_datetime = param.Date(
         default=date_bounds[0],
         label="Date and time (UTC)",
         doc=f"Select dates between {date_bounds[0]} and {date_bounds[1]}",
         bounds=date_bounds,
+        precedence=3,
     )
     url_mjd = param.Number(default=None)
     widget_tier = param.Selector(
@@ -167,6 +169,7 @@ class Scheduler(param.Parameterized):
         objects=[""],
         label="Tier",
         doc="The label for the first index into the CoreScheduler.survey_lists.",
+        precedence=4,
     )
     survey_map = param.Selector(
         default="reward",
@@ -179,6 +182,7 @@ class Scheduler(param.Parameterized):
         label="Map resolution (nside)",
         doc="",
     )
+
     color_palette = param.Selector(default=DEFAULT_COLOR_PALETTE, objects=COLOR_PALETTES, doc="")
     summary_widget = param.Parameter(default=None, doc="")
     reward_widget = param.Parameter(default=None, doc="")
@@ -1449,6 +1453,27 @@ def __init__(self, data_dir=None):
             self.param["scheduler_fname"].update(path=f"{data_dir}/*scheduler*.p*")
 
 
+class USDFScheduler(Scheduler):
+    """A Parametrized container for parameters, data, and panel objects for the
+    scheduler dashboard.
+    """
+
+    # Param parameters that are modifiable by user actions.
+    scheduler_fname_doc = """Recent pickles from USDF
+    """
+
+    scheduler_fname = param.Selector(
+        default="file1", objects=["file1", "file2"], doc=scheduler_fname_doc, precedence=2
+    )
+
+    pickles_date = param.Date(
+        default=datetime.now(), label="Pickles Date", doc="Select date to load pickles for", precedence=1
+    )
+
+    def __init__(self):
+        super().__init__()
+
+
 # ------------------------------------------------------------ Create dashboard
 
 
@@ -1476,6 +1501,7 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
 
     from_urls = False
     data_dir = None
+    from_usdf = False
 
     if "data_from_urls" in kwargs.keys():
         from_urls = kwargs["data_from_urls"]
@@ -1485,8 +1511,14 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
         data_dir = kwargs["data_dir"]
         del kwargs["data_dir"]
 
+    if "usdf" in kwargs.keys():
+        from_usdf = kwargs["usdf"]
+        del kwargs["usdf"]
+
     scheduler = None
     data_loading_widgets = {}
+    data_loading_parameters = ["scheduler_fname", "widget_datetime", "widget_tier"]
+
     # Accept pickle files from url or any path.
     if from_urls:
         scheduler = Scheduler()
@@ -1516,6 +1548,14 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
             },
             "widget_datetime": pn.widgets.DatetimePicker,
         }
+    # Load pickles from USDF S3 bucket
+    elif from_usdf:
+        scheduler = USDFScheduler()
+        data_loading_parameters = ["scheduler_fname", "pickles_date", "widget_datetime", "widget_tier"]
+        data_loading_widgets = {
+            "pickles_date": pn.widgets.DatetimePicker,
+            "widget_datetime": pn.widgets.DatetimePicker,
+        }
 
     # Restrict files to data_directory.
     else:
@@ -1578,16 +1618,16 @@ def handle_reload_pickle(event):
         styles={"background": "#048b8c"},
     )
     # Parameter inputs (pickle, widget_datetime, tier).
-    sched_app[8:30, 0:21] = pn.Param(
+    sched_app[8:36, 0:21] = pn.Param(
         scheduler,
-        parameters=["scheduler_fname", "widget_datetime", "widget_tier"],
+        parameters=data_loading_parameters,
         widgets=data_loading_widgets,
         name="Select pickle file, date and tier.",
     )
     # Reset button.
-    sched_app[30:36, 3:15] = pn.Row(reset_button)
+    sched_app[36:42, 3:15] = pn.Row(reset_button)
     # Survey rewards table and header.
-    sched_app[8:36, 21:67] = pn.Row(
+    sched_app[8:42, 21:67] = pn.Row(
         pn.Spacer(width=10),
         pn.Column(
             pn.Spacer(height=10),
@@ -1602,7 +1642,7 @@ def handle_reload_pickle(event):
         # sizing_mode="stretch_height",
     )
     # Reward table and header.
-    sched_app[36:87, 0:67] = pn.Row(
+    sched_app[42:87, 0:67] = pn.Row(
         pn.Spacer(width=10),
         pn.Column(
             pn.Spacer(height=10),
@@ -1669,11 +1709,20 @@ def parse_arguments():
         help="Let the user specify URLs from which to load data. THIS IS NOT SECURE.",
     )
 
+    parser.add_argument(
+        "--usdf",
+        action="store_true",
+        help="Loads pickle files from the data directory in USDF",
+    )
+
     args = parser.parse_args()
 
     if len(glob(args.data_dir)) == 0 and not args.data_from_urls:
         args.data_dir = PACKAGE_DATA_DIR
 
+    if args.usdf and len(glob(USDF_DATA_DIR)) == 0:
+        args.data_dir = PACKAGE_DATA_DIR
+
     scheduler_app_params = args.__dict__
 
     return scheduler_app_params

From acf0e721dd3f30aae69f6c1f62189b4a29ad9e29 Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@swin.edu.au>
Date: Tue, 19 Mar 2024 22:22:00 +1100
Subject: [PATCH 02/21] use ResourcePath from lsst.resources library to load
 files

---
 schedview/app/scheduler_dashboard/scheduler_dashboard.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index 8ddeb7f8..c628ac5f 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -44,6 +44,7 @@
 from astropy.utils.exceptions import AstropyWarning
 from bokeh.models import ColorBar, LinearColorMapper
 from bokeh.models.widgets.tables import BooleanFormatter, HTMLTemplateFormatter, NumberFormatter
+from lsst.resources import ResourcePath
 from pandas import Timestamp
 from panel.io.loading import start_loading_spinner, stop_loading_spinner
 from pytz import timezone
@@ -608,7 +609,12 @@ def read_scheduler(self):
             self._debugging_message = "Starting to load scheduler."
             pn.state.notifications.info("Scheduler loading...", duration=0)
 
-            (scheduler, conditions) = schedview.collect.scheduler_pickle.read_scheduler(self.scheduler_fname)
+            scheduler_resource_path = ResourcePath(self.scheduler_fname)
+            with scheduler_resource_path.as_local() as local_scheduler_resource:
+                (scheduler, conditions) = schedview.collect.scheduler_pickle.read_scheduler(
+                    local_scheduler_resource.ospath
+                )
+
             self._scheduler = scheduler
             self._conditions = conditions
 

From 97ef930ba115b1c963188d2970365b3e11b7c42c Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome001.sdf.slac.stanford.edu>
Date: Thu, 4 Apr 2024 06:22:40 -0700
Subject: [PATCH 03/21] read scheduler from a mock dataframe till async issue
 is resolved, turn off threading when reading the pickle using resource path

---
 .../scheduler_dashboard.py                    |  42 +++++-
 schedview/app/scheduler_dashboard/utils.py    | 120 ++++++++++++++++++
 schedview/collect/scheduler_pickle.py         |  61 +++++----
 3 files changed, 191 insertions(+), 32 deletions(-)
 create mode 100644 schedview/app/scheduler_dashboard/utils.py

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index c628ac5f..aefb0e66 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -34,24 +34,27 @@
 from datetime import datetime
 from glob import glob
 from zoneinfo import ZoneInfo
+from concurrent.futures import ThreadPoolExecutor
 
 import bokeh
 import numpy as np
 import panel as pn
 import param
 import rubin_scheduler.site_models
-from astropy.time import Time
+from astropy.time import Time, TimeDelta
 from astropy.utils.exceptions import AstropyWarning
 from bokeh.models import ColorBar, LinearColorMapper
 from bokeh.models.widgets.tables import BooleanFormatter, HTMLTemplateFormatter, NumberFormatter
-from lsst.resources import ResourcePath
 from pandas import Timestamp
 from panel.io.loading import start_loading_spinner, stop_loading_spinner
 from pytz import timezone
+import astropy.units as u
 
 # For the conditions.mjd bugfix
 from rubin_scheduler.scheduler.model_observatory import ModelObservatory
 from rubin_scheduler.skybrightness_pre.sky_model_pre import SkyModelPre
+from lsst_efd_client import EfdClient
+from lsst.resources import ResourcePath
 
 import schedview
 import schedview.collect.scheduler_pickle
@@ -59,6 +62,8 @@
 import schedview.compute.survey
 import schedview.param
 import schedview.plot.survey
+from util.efd_sched import localize_scheduler_url, query_night_schedulers
+from schedview.app.scheduler_dashboard.utils import mock_schedulers_df
 
 # Filter astropy warning that's filling the terminal with every update.
 warnings.filterwarnings("ignore", category=AstropyWarning)
@@ -70,7 +75,7 @@
 DEFAULT_COLOR_PALETTE = "Viridis256"
 DEFAULT_NSIDE = 16
 PACKAGE_DATA_DIR = importlib.resources.files("schedview.data").as_posix()
-USDF_DATA_DIR = "/sdf/group/rubin/web_data/sim-data/schedview"
+USDF_DATA_DIR = "s3://rubin:"
 
 
 pn.extension(
@@ -609,7 +614,10 @@ def read_scheduler(self):
             self._debugging_message = "Starting to load scheduler."
             pn.state.notifications.info("Scheduler loading...", duration=0)
 
+            os.environ["LSST_DISABLE_BUCKET_VALIDATION"] = "1"
             scheduler_resource_path = ResourcePath(self.scheduler_fname)
+            scheduler_resource_path.use_threads = False
+            assert not scheduler_resource_path._environ_use_threads
             with scheduler_resource_path.as_local() as local_scheduler_resource:
                 (scheduler, conditions) = schedview.collect.scheduler_pickle.read_scheduler(
                     local_scheduler_resource.ospath
@@ -1469,7 +1477,7 @@ class USDFScheduler(Scheduler):
     """
 
     scheduler_fname = param.Selector(
-        default="file1", objects=["file1", "file2"], doc=scheduler_fname_doc, precedence=2
+        default="", objects=["", "s3://rubin:rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:23:14.427.p"], doc=scheduler_fname_doc, precedence=2
     )
 
     pickles_date = param.Date(
@@ -1480,6 +1488,25 @@ def __init__(self):
         super().__init__()
 
 
+    async def query_schedulers(self, selected_time):
+        print(f"{selected_time.date()}")
+        # time_window=TimeDelta(2 * u.second)
+        # desired_time = Time(selected_time)
+        # start_time = desired_time - (time_window / 2)
+        # end_time = desired_time + (time_window / 2)
+
+        # print(f"start time {start_time}")
+        # print(f"end time {end_time}")
+
+        # efd_client = EfdClient("usdf_efd")
+        # topic = "lsst.sal.Scheduler.logevent_largeFileObjectAvailable"
+        # fields = ["url"]
+        # scheduler_urls = await efd_client.select_time_series(topic, fields, start_time, end_time)
+        # scheduler_urls.index.name = "time"
+        # scheduler_urls = scheduler_urls.reset_index()
+        # scheduler_urls = await query_night_schedulers(str(selected_time.date()))
+        scheduler_urls = mock_schedulers_df()
+        return scheduler_urls
 # ------------------------------------------------------------ Create dashboard
 
 
@@ -1563,6 +1590,13 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
             "widget_datetime": pn.widgets.DatetimePicker,
         }
 
+        @pn.depends(selected_time=scheduler.param.pickles_date, watch=True)
+        async def get_scheduler_list(selected_time):
+            os.environ["LSST_DISABLE_BUCKET_VALIDATION"] = "1"
+            schedulers = await scheduler.query_schedulers(selected_time)
+            scheduler.param["scheduler_fname"].objects = schedulers['url']
+            # scheduler.scheduler_fname = schedulers[0] # "s3://rubin:rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:23:14.427.p"
+
     # Restrict files to data_directory.
     else:
         scheduler = RestrictedFilesScheduler(data_dir=data_dir)
diff --git a/schedview/app/scheduler_dashboard/utils.py b/schedview/app/scheduler_dashboard/utils.py
new file mode 100644
index 00000000..3bc96a00
--- /dev/null
+++ b/schedview/app/scheduler_dashboard/utils.py
@@ -0,0 +1,120 @@
+import argparse
+import asyncio
+import lzma
+import os
+import pickle
+import re
+
+import astropy.units as u
+from astropy.time import Time, TimeDelta
+import pandas as pd
+from lsst.resources import ResourcePath
+from lsst_efd_client import EfdClient
+from rubin_scheduler.utils import Site
+
+LOCAL_ROOT_URI = {"usdf": "s3://rubin:", "summit": "https://s3.cp.lsst.org/"}
+
+
+async def query_schedulers_in_window(desired_time, efd="usdf_efd", time_window=TimeDelta(2 * u.second)):
+    """Query the EFD for scheduler URLs within a given time window.
+
+    Parameters
+    ----------
+    desired_time : `astropy.time.Time`
+        The central time to query the event service.
+    efd : `str`, optional
+        The name of the EFD to connect to (default is 'usdf_efd').
+    time_window : `astropy.time.TimeDelta`, optional
+        The size of the time window to search for scheduler URLs
+        (default is 2 seconds).
+
+    Returns
+    -------
+    scheduler_urls : `pandas.DataFrame`
+        A DataFrame containing the scheduler snapshot times URLs.
+    """
+    start_time = desired_time - (time_window / 2)
+    end_time = desired_time + (time_window / 2)
+
+    efd_client = EfdClient(efd)
+    topic = "lsst.sal.Scheduler.logevent_largeFileObjectAvailable"
+    fields = ["url"]
+    scheduler_urls = await efd_client.select_time_series(topic, fields, start_time, end_time)
+    scheduler_urls.index.name = "time"
+    scheduler_urls = scheduler_urls.reset_index()
+    return scheduler_urls
+
+
+async def query_night_schedulers(night, efd="usdf_efd"):
+    """Query the EFD for the night schedulers
+
+    Parameters
+    ----------
+    night : `str`, 'float', or `astropy.time.Time`
+        The night to query, in YYYY-MM-DD format
+    efd : `str`
+        The name of the EFD to query, usdf_efd or summit_efd
+        Defaults to usdf_efd
+
+    Returns
+    -------
+    schedulers : `list`
+        A list of the schedulers for the night
+    """
+
+    try:
+        reference_time = Time(night)
+    except ValueError:
+        if isinstance(night, int):
+            reference_time = Time(night, format="mjd")
+        else:
+            raise ValueError("night must be a valid date in YYYY-MM-DD format")
+
+    # The offset by longitude moves to local solar time.
+    # The offset of 1 makes the night specificed refer to the
+    # local date corresponding to sunset.
+    local_midnight = Time(reference_time.mjd + 1 - Site("LSST").longitude / 360, format="mjd")
+    time_window = TimeDelta(1, format="jd")
+    scheduler_urls = await query_schedulers_in_window(local_midnight, efd=efd, time_window=time_window)
+    return scheduler_urls
+
+def localize_scheduler_url(scheduler_url, site="usdf"):
+    """Localizes the scheduler URL for a given site.
+
+    Parameters
+    ----------
+    scheduler_url : `str`
+        The URL of the scheduler to be localized.
+    site : `str` , optional
+        The site to which the scheduler URL should be localized.
+        Defaults to 'usdf'.
+
+    Returns
+    -------
+    scheduler_url : `str`
+        The localized URL of the scheduler.
+    """
+    # If we don't have a canonical root for the site, just return
+    # the original
+    if site not in LOCAL_ROOT_URI:
+        return scheduler_url
+
+    original_scheduler_resource_path = ResourcePath(scheduler_url)
+    scheduler_path = original_scheduler_resource_path.path.lstrip("/")
+    root_uri = LOCAL_ROOT_URI[site]
+    scheduler_url = f"{root_uri}{scheduler_path}"
+    return scheduler_url
+
+def mock_schedulers_df():
+    data = [['2024-03-12 00:30:40.565731+00:00', 'https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:1/Scheduler:1/2024/03/11/Scheduler:1_Scheduler:1_2024-03-12T00:31:16.956.p'],
+            ['2024-03-12 00:32:29.309646+00:00', 'https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T00:33:05.127.p'],
+            ['2024-03-12 00:57:23.035425+00:00', 'https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T00:57:58.876.p'],
+            ['2024-03-12 01:06:34.583256+00:00', 'https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:07:10.474.p'],
+            ['2024-03-12 01:15:09.624082+00:00', 'https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:15:45.472.p'],
+            ['2024-03-12 01:22:38.678879+00:00', 'https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:23:14.427.p'],
+            ['2024-03-12 01:34:39.499348+00:00', 'https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:35:15.393.p']]
+
+    df = pd.DataFrame(data, columns=['time', 'url'])
+    df = df.sort_index(ascending=False)
+    df['url'] = df['url'].apply(lambda x: localize_scheduler_url(x))
+    return df
diff --git a/schedview/collect/scheduler_pickle.py b/schedview/collect/scheduler_pickle.py
index 3faef3c3..04e5020c 100644
--- a/schedview/collect/scheduler_pickle.py
+++ b/schedview/collect/scheduler_pickle.py
@@ -10,6 +10,7 @@
 import urllib.request
 from pathlib import Path
 from tempfile import TemporaryDirectory
+from lsst.resources import ResourcePath
 
 from rubin_scheduler.scheduler.model_observatory import ModelObservatory
 
@@ -80,34 +81,38 @@ def read_scheduler(file_name_or_url=None):
     conditions : `rubin_scheduler.scheduler.features.Conditions`
         An instance of a rubin_scheduler conditions object.
     """
-    if file_name_or_url is None:
-        file_name_or_url = PICKLE_FNAME
-
-    if file_name_or_url is None:
-        file_name_or_url = sample_pickle()
-
-    if Path(file_name_or_url).is_file():
-        scheduler, conditions = read_local_scheduler_pickle(file_name_or_url)
-    else:
-        # If we didn't have to decompress it, we could use urlopen instead
-        # of downloading a local copy. But, it can be compressed, so we need
-        # to use gzip.open to open it.
-        with TemporaryDirectory() as directory:
-            with urllib.request.urlopen(file_name_or_url) as url_io:
-                content = url_io.read()
-
-            # Infer a file name
-            parsed_url = urllib.parse.urlparse(file_name_or_url)
-            origin_path = Path(parsed_url.path)
-            origin_name = origin_path.name
-            name = origin_name if len(origin_name) > 0 else "scheduler.pickle"
-            path = Path(directory).joinpath(name)
-
-            with open(path, "wb") as file_io:
-                file_io.write(content)
-
-            scheduler, conditions = read_local_scheduler_pickle(str(path))
-
+    # if file_name_or_url is None:
+    #     file_name_or_url = PICKLE_FNAME
+
+    # if file_name_or_url is None:
+    #     file_name_or_url = sample_pickle()
+
+    # if Path(file_name_or_url).is_file():
+    #     scheduler, conditions = read_local_scheduler_pickle(file_name_or_url)
+    # else:
+    #     # If we didn't have to decompress it, we could use urlopen instead
+    #     # of downloading a local copy. But, it can be compressed, so we need
+    #     # to use gzip.open to open it.
+    #     with TemporaryDirectory() as directory:
+    #         with urllib.request.urlopen(file_name_or_url) as url_io:
+    #             content = url_io.read()
+
+    #         # Infer a file name
+    #         parsed_url = urllib.parse.urlparse(file_name_or_url)
+    #         origin_path = Path(parsed_url.path)
+    #         origin_name = origin_path.name
+    #         name = origin_name if len(origin_name) > 0 else "scheduler.pickle"
+    #         path = Path(directory).joinpath(name)
+
+    #         with open(path, "wb") as file_io:
+    #             file_io.write(content)
+
+    #         scheduler, conditions = read_local_scheduler_pickle(str(path))
+    scheduler_resource_path = ResourcePath(file_name_or_url)
+    with scheduler_resource_path.as_local() as local_scheduler_resource:
+        (scheduler, conditions) = read_local_scheduler_pickle(
+            local_scheduler_resource.ospath
+        )
     return scheduler, conditions
 
 

From 0d102e9b14f5bc1d54561031c90cadb0345546f0 Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome001.sdf.slac.stanford.edu>
Date: Thu, 4 Apr 2024 06:42:18 -0700
Subject: [PATCH 04/21] pre-commit changes

---
 .../scheduler_dashboard.py                    | 37 +++++--------
 schedview/app/scheduler_dashboard/utils.py    | 54 ++++++++++++-------
 schedview/collect/scheduler_pickle.py         | 38 ++-----------
 3 files changed, 51 insertions(+), 78 deletions(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index aefb0e66..62f1fc61 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -34,27 +34,24 @@
 from datetime import datetime
 from glob import glob
 from zoneinfo import ZoneInfo
-from concurrent.futures import ThreadPoolExecutor
 
 import bokeh
 import numpy as np
 import panel as pn
 import param
 import rubin_scheduler.site_models
-from astropy.time import Time, TimeDelta
+from astropy.time import Time
 from astropy.utils.exceptions import AstropyWarning
 from bokeh.models import ColorBar, LinearColorMapper
 from bokeh.models.widgets.tables import BooleanFormatter, HTMLTemplateFormatter, NumberFormatter
+from lsst.resources import ResourcePath
 from pandas import Timestamp
 from panel.io.loading import start_loading_spinner, stop_loading_spinner
 from pytz import timezone
-import astropy.units as u
 
 # For the conditions.mjd bugfix
 from rubin_scheduler.scheduler.model_observatory import ModelObservatory
 from rubin_scheduler.skybrightness_pre.sky_model_pre import SkyModelPre
-from lsst_efd_client import EfdClient
-from lsst.resources import ResourcePath
 
 import schedview
 import schedview.collect.scheduler_pickle
@@ -62,7 +59,6 @@
 import schedview.compute.survey
 import schedview.param
 import schedview.plot.survey
-from util.efd_sched import localize_scheduler_url, query_night_schedulers
 from schedview.app.scheduler_dashboard.utils import mock_schedulers_df
 
 # Filter astropy warning that's filling the terminal with every update.
@@ -1477,7 +1473,13 @@ class USDFScheduler(Scheduler):
     """
 
     scheduler_fname = param.Selector(
-        default="", objects=["", "s3://rubin:rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:23:14.427.p"], doc=scheduler_fname_doc, precedence=2
+        default="",
+        objects=[
+            "",
+            "s3://rubin:rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:23:14.427.p",
+        ],
+        doc=scheduler_fname_doc,
+        precedence=2,
     )
 
     pickles_date = param.Date(
@@ -1487,26 +1489,11 @@ class USDFScheduler(Scheduler):
     def __init__(self):
         super().__init__()
 
-
     async def query_schedulers(self, selected_time):
-        print(f"{selected_time.date()}")
-        # time_window=TimeDelta(2 * u.second)
-        # desired_time = Time(selected_time)
-        # start_time = desired_time - (time_window / 2)
-        # end_time = desired_time + (time_window / 2)
-
-        # print(f"start time {start_time}")
-        # print(f"end time {end_time}")
-
-        # efd_client = EfdClient("usdf_efd")
-        # topic = "lsst.sal.Scheduler.logevent_largeFileObjectAvailable"
-        # fields = ["url"]
-        # scheduler_urls = await efd_client.select_time_series(topic, fields, start_time, end_time)
-        # scheduler_urls.index.name = "time"
-        # scheduler_urls = scheduler_urls.reset_index()
-        # scheduler_urls = await query_night_schedulers(str(selected_time.date()))
         scheduler_urls = mock_schedulers_df()
         return scheduler_urls
+
+
 # ------------------------------------------------------------ Create dashboard
 
 
@@ -1594,7 +1581,7 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
         async def get_scheduler_list(selected_time):
             os.environ["LSST_DISABLE_BUCKET_VALIDATION"] = "1"
             schedulers = await scheduler.query_schedulers(selected_time)
-            scheduler.param["scheduler_fname"].objects = schedulers['url']
+            scheduler.param["scheduler_fname"].objects = schedulers["url"]
             # scheduler.scheduler_fname = schedulers[0] # "s3://rubin:rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:23:14.427.p"
 
     # Restrict files to data_directory.
diff --git a/schedview/app/scheduler_dashboard/utils.py b/schedview/app/scheduler_dashboard/utils.py
index 3bc96a00..670114b5 100644
--- a/schedview/app/scheduler_dashboard/utils.py
+++ b/schedview/app/scheduler_dashboard/utils.py
@@ -1,13 +1,6 @@
-import argparse
-import asyncio
-import lzma
-import os
-import pickle
-import re
-
 import astropy.units as u
-from astropy.time import Time, TimeDelta
 import pandas as pd
+from astropy.time import Time, TimeDelta
 from lsst.resources import ResourcePath
 from lsst_efd_client import EfdClient
 from rubin_scheduler.utils import Site
@@ -78,6 +71,7 @@ async def query_night_schedulers(night, efd="usdf_efd"):
     scheduler_urls = await query_schedulers_in_window(local_midnight, efd=efd, time_window=time_window)
     return scheduler_urls
 
+
 def localize_scheduler_url(scheduler_url, site="usdf"):
     """Localizes the scheduler URL for a given site.
 
@@ -105,16 +99,40 @@ def localize_scheduler_url(scheduler_url, site="usdf"):
     scheduler_url = f"{root_uri}{scheduler_path}"
     return scheduler_url
 
+
 def mock_schedulers_df():
-    data = [['2024-03-12 00:30:40.565731+00:00', 'https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:1/Scheduler:1/2024/03/11/Scheduler:1_Scheduler:1_2024-03-12T00:31:16.956.p'],
-            ['2024-03-12 00:32:29.309646+00:00', 'https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T00:33:05.127.p'],
-            ['2024-03-12 00:57:23.035425+00:00', 'https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T00:57:58.876.p'],
-            ['2024-03-12 01:06:34.583256+00:00', 'https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:07:10.474.p'],
-            ['2024-03-12 01:15:09.624082+00:00', 'https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:15:45.472.p'],
-            ['2024-03-12 01:22:38.678879+00:00', 'https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:23:14.427.p'],
-            ['2024-03-12 01:34:39.499348+00:00', 'https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:35:15.393.p']]
-
-    df = pd.DataFrame(data, columns=['time', 'url'])
+    data = [
+        [
+            "2024-03-12 00:30:40.565731+00:00",
+            "https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:1/Scheduler:1/2024/03/11/Scheduler:1_Scheduler:1_2024-03-12T00:31:16.956.p",
+        ],
+        [
+            "2024-03-12 00:32:29.309646+00:00",
+            "https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T00:33:05.127.p",
+        ],
+        [
+            "2024-03-12 00:57:23.035425+00:00",
+            "https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T00:57:58.876.p",
+        ],
+        [
+            "2024-03-12 01:06:34.583256+00:00",
+            "https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:07:10.474.p",
+        ],
+        [
+            "2024-03-12 01:15:09.624082+00:00",
+            "https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:15:45.472.p",
+        ],
+        [
+            "2024-03-12 01:22:38.678879+00:00",
+            "https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:23:14.427.p",
+        ],
+        [
+            "2024-03-12 01:34:39.499348+00:00",
+            "https://s3.cp.lsst.org/rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:35:15.393.p",
+        ],
+    ]
+
+    df = pd.DataFrame(data, columns=["time", "url"])
     df = df.sort_index(ascending=False)
-    df['url'] = df['url'].apply(lambda x: localize_scheduler_url(x))
+    df["url"] = df["url"].apply(lambda x: localize_scheduler_url(x))
     return df
diff --git a/schedview/collect/scheduler_pickle.py b/schedview/collect/scheduler_pickle.py
index 04e5020c..86f11679 100644
--- a/schedview/collect/scheduler_pickle.py
+++ b/schedview/collect/scheduler_pickle.py
@@ -6,12 +6,8 @@
 import lzma
 import os
 import pickle
-import urllib
-import urllib.request
-from pathlib import Path
-from tempfile import TemporaryDirectory
-from lsst.resources import ResourcePath
 
+from lsst.resources import ResourcePath
 from rubin_scheduler.scheduler.model_observatory import ModelObservatory
 
 try:
@@ -81,38 +77,10 @@ def read_scheduler(file_name_or_url=None):
     conditions : `rubin_scheduler.scheduler.features.Conditions`
         An instance of a rubin_scheduler conditions object.
     """
-    # if file_name_or_url is None:
-    #     file_name_or_url = PICKLE_FNAME
-
-    # if file_name_or_url is None:
-    #     file_name_or_url = sample_pickle()
-
-    # if Path(file_name_or_url).is_file():
-    #     scheduler, conditions = read_local_scheduler_pickle(file_name_or_url)
-    # else:
-    #     # If we didn't have to decompress it, we could use urlopen instead
-    #     # of downloading a local copy. But, it can be compressed, so we need
-    #     # to use gzip.open to open it.
-    #     with TemporaryDirectory() as directory:
-    #         with urllib.request.urlopen(file_name_or_url) as url_io:
-    #             content = url_io.read()
-
-    #         # Infer a file name
-    #         parsed_url = urllib.parse.urlparse(file_name_or_url)
-    #         origin_path = Path(parsed_url.path)
-    #         origin_name = origin_path.name
-    #         name = origin_name if len(origin_name) > 0 else "scheduler.pickle"
-    #         path = Path(directory).joinpath(name)
-
-    #         with open(path, "wb") as file_io:
-    #             file_io.write(content)
-
-    #         scheduler, conditions = read_local_scheduler_pickle(str(path))
+
     scheduler_resource_path = ResourcePath(file_name_or_url)
     with scheduler_resource_path.as_local() as local_scheduler_resource:
-        (scheduler, conditions) = read_local_scheduler_pickle(
-            local_scheduler_resource.ospath
-        )
+        (scheduler, conditions) = read_local_scheduler_pickle(local_scheduler_resource.ospath)
     return scheduler, conditions
 
 

From aecb7ce1ec2a8d7106fdeb8e44c6f57acdb96e06 Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome002.sdf.slac.stanford.edu>
Date: Sun, 14 Apr 2024 05:27:54 -0700
Subject: [PATCH 05/21] update the query function to query times from the start
 of the night till selected timestamp

---
 .../scheduler_dashboard.py                    | 20 ++--
 schedview/app/scheduler_dashboard/utils.py    | 98 +++++++++----------
 2 files changed, 63 insertions(+), 55 deletions(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index 62f1fc61..0008f277 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -59,7 +59,7 @@
 import schedview.compute.survey
 import schedview.param
 import schedview.plot.survey
-from schedview.app.scheduler_dashboard.utils import mock_schedulers_df
+from schedview.app.scheduler_dashboard.utils import query_night_schedulers
 
 # Filter astropy warning that's filling the terminal with every update.
 warnings.filterwarnings("ignore", category=AstropyWarning)
@@ -1490,7 +1490,16 @@ def __init__(self):
         super().__init__()
 
     async def query_schedulers(self, selected_time):
-        scheduler_urls = mock_schedulers_df()
+        selected_time = Time(
+            Timestamp(
+                selected_time,
+                tzinfo=ZoneInfo(DEFAULT_TIMEZONE),
+            )
+        )
+        self.show_loading_indicator = True
+        scheduler_urls = await query_night_schedulers(selected_time)
+        self.show_loading_indicator = False
+        # scheduler_urls = await get_top_n_schedulers()
         return scheduler_urls
 
 
@@ -1581,8 +1590,7 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
         async def get_scheduler_list(selected_time):
             os.environ["LSST_DISABLE_BUCKET_VALIDATION"] = "1"
             schedulers = await scheduler.query_schedulers(selected_time)
-            scheduler.param["scheduler_fname"].objects = schedulers["url"]
-            # scheduler.scheduler_fname = schedulers[0] # "s3://rubin:rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:23:14.427.p"
+            scheduler.param["scheduler_fname"].objects = schedulers
 
     # Restrict files to data_directory.
     else:
@@ -1762,7 +1770,7 @@ def main():
     if "SCHEDULER_PORT" in os.environ:
         scheduler_port = int(os.environ["SCHEDULER_PORT"])
     else:
-        scheduler_port = 8080
+        scheduler_port = 8888
 
     assets_dir = os.path.join(importlib.resources.files("schedview"), "app", "scheduler_dashboard", "assets")
 
@@ -1781,7 +1789,7 @@ def scheduler_app_with_params():
         prefix=prefix,
         start=True,
         autoreload=True,
-        threaded=True,
+        # threaded=True,
         static_dirs={"assets": assets_dir},
     )
 
diff --git a/schedview/app/scheduler_dashboard/utils.py b/schedview/app/scheduler_dashboard/utils.py
index 670114b5..26c420cd 100644
--- a/schedview/app/scheduler_dashboard/utils.py
+++ b/schedview/app/scheduler_dashboard/utils.py
@@ -1,49 +1,20 @@
-import astropy.units as u
 import pandas as pd
 from astropy.time import Time, TimeDelta
 from lsst.resources import ResourcePath
 from lsst_efd_client import EfdClient
+from pandas import Timestamp
+from pytz import timezone
 from rubin_scheduler.utils import Site
 
 LOCAL_ROOT_URI = {"usdf": "s3://rubin:", "summit": "https://s3.cp.lsst.org/"}
 
 
-async def query_schedulers_in_window(desired_time, efd="usdf_efd", time_window=TimeDelta(2 * u.second)):
-    """Query the EFD for scheduler URLs within a given time window.
-
-    Parameters
-    ----------
-    desired_time : `astropy.time.Time`
-        The central time to query the event service.
-    efd : `str`, optional
-        The name of the EFD to connect to (default is 'usdf_efd').
-    time_window : `astropy.time.TimeDelta`, optional
-        The size of the time window to search for scheduler URLs
-        (default is 2 seconds).
-
-    Returns
-    -------
-    scheduler_urls : `pandas.DataFrame`
-        A DataFrame containing the scheduler snapshot times URLs.
-    """
-    start_time = desired_time - (time_window / 2)
-    end_time = desired_time + (time_window / 2)
-
-    efd_client = EfdClient(efd)
-    topic = "lsst.sal.Scheduler.logevent_largeFileObjectAvailable"
-    fields = ["url"]
-    scheduler_urls = await efd_client.select_time_series(topic, fields, start_time, end_time)
-    scheduler_urls.index.name = "time"
-    scheduler_urls = scheduler_urls.reset_index()
-    return scheduler_urls
-
-
-async def query_night_schedulers(night, efd="usdf_efd"):
+async def query_night_schedulers(reference_time_utc, efd="usdf_efd"):
     """Query the EFD for the night schedulers
 
     Parameters
     ----------
-    night : `str`, 'float', or `astropy.time.Time`
+    selected_time : `str`, 'float', or `astropy.time.Time`
         The night to query, in YYYY-MM-DD format
     efd : `str`
         The name of the EFD to query, usdf_efd or summit_efd
@@ -55,21 +26,37 @@ async def query_night_schedulers(night, efd="usdf_efd"):
         A list of the schedulers for the night
     """
 
-    try:
-        reference_time = Time(night)
-    except ValueError:
-        if isinstance(night, int):
-            reference_time = Time(night, format="mjd")
-        else:
-            raise ValueError("night must be a valid date in YYYY-MM-DD format")
-
-    # The offset by longitude moves to local solar time.
-    # The offset of 1 makes the night specificed refer to the
-    # local date corresponding to sunset.
-    local_midnight = Time(reference_time.mjd + 1 - Site("LSST").longitude / 360, format="mjd")
-    time_window = TimeDelta(1, format="jd")
-    scheduler_urls = await query_schedulers_in_window(local_midnight, efd=efd, time_window=time_window)
-    return scheduler_urls
+    # reference_time_utc = Time(selected_time)
+    # print(f"reference_time_utc {reference_time_utc}")
+    # tz = timezone('Etc/GMT+12')
+    # reference_timestamp_utc_12 =
+    # Timestamp(reference_time_utc.to_datetime(timezone=tz))
+    # reference_date_utc_12 = reference_timestamp_utc_12.date()
+    # print(f"reference_date_utc_12 {reference_date_utc_12}")
+    # print(f"reference_timestamp_utc_12 {reference_timestamp_utc_12}")
+    # end_time = Time(str(reference_timestamp_utc_12))
+    # start_time = Time(Timestamp(str(reference_date_utc_12)))
+
+    tz = timezone("Etc/GMT+12")
+    reference_time_utc_12 = Timestamp(reference_time_utc.to_datetime(timezone=tz))
+    night = Time(str(reference_time_utc_12.date()))
+    local_midnight = Time(night.mjd + 1 - Site("LSST").longitude / 360, format="mjd")
+    print(f"local midnight {local_midnight.to_datetime()}")
+    start_time = local_midnight - TimeDelta(0.5, format="jd")
+    end_time = reference_time_utc
+    print(f"star_time {start_time.to_datetime()}")
+    print(f"end_time {end_time}")
+    efd_client = EfdClient(efd)
+    topic = "lsst.sal.Scheduler.logevent_largeFileObjectAvailable"
+    fields = ["url"]
+    scheduler_urls = await efd_client.select_time_series(topic, fields, start_time, end_time)
+    if not scheduler_urls.empty:
+        scheduler_urls.index.name = "time"
+        scheduler_urls = scheduler_urls.reset_index()
+        scheduler_urls = scheduler_urls.sort_index(ascending=False)
+        scheduler_urls["url"] = scheduler_urls["url"].apply(lambda x: localize_scheduler_url(x))
+        return scheduler_urls["url"]
+    return []
 
 
 def localize_scheduler_url(scheduler_url, site="usdf"):
@@ -135,4 +122,17 @@ def mock_schedulers_df():
     df = pd.DataFrame(data, columns=["time", "url"])
     df = df.sort_index(ascending=False)
     df["url"] = df["url"].apply(lambda x: localize_scheduler_url(x))
-    return df
+    return df["url"]
+
+
+async def get_top_n_schedulers(efd="usdf_efd"):
+    sync_client = EfdClient(efd, db_name="efd")
+    topic = "lsst.sal.Scheduler.logevent_largeFileObjectAvailable"
+    fields = ["url"]
+    scheduler_urls = await sync_client.select_top_n(topic, fields, 10)
+    scheduler_urls.index.name = "time"
+    scheduler_urls = scheduler_urls.reset_index()
+
+    scheduler_urls = scheduler_urls.sort_index(ascending=False)
+    scheduler_urls["url"] = scheduler_urls["url"].apply(lambda x: localize_scheduler_url(x))
+    return scheduler_urls["url"]

From 7fc46d05388940d2254f5f2fcacbcac6ed8df008 Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome002.sdf.slac.stanford.edu>
Date: Sun, 14 Apr 2024 05:35:33 -0700
Subject: [PATCH 06/21] clean code

---
 .../scheduler_dashboard.py                    |  6 +----
 schedview/app/scheduler_dashboard/utils.py    | 27 +++++++------------
 2 files changed, 10 insertions(+), 23 deletions(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index 0008f277..b66a2cfc 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -1474,10 +1474,7 @@ class USDFScheduler(Scheduler):
 
     scheduler_fname = param.Selector(
         default="",
-        objects=[
-            "",
-            "s3://rubin:rubinobs-lfa-cp/Scheduler:2/Scheduler:2/2024/03/11/Scheduler:2_Scheduler:2_2024-03-12T01:23:14.427.p",
-        ],
+        objects=[],
         doc=scheduler_fname_doc,
         precedence=2,
     )
@@ -1499,7 +1496,6 @@ async def query_schedulers(self, selected_time):
         self.show_loading_indicator = True
         scheduler_urls = await query_night_schedulers(selected_time)
         self.show_loading_indicator = False
-        # scheduler_urls = await get_top_n_schedulers()
         return scheduler_urls
 
 
diff --git a/schedview/app/scheduler_dashboard/utils.py b/schedview/app/scheduler_dashboard/utils.py
index 26c420cd..259400e8 100644
--- a/schedview/app/scheduler_dashboard/utils.py
+++ b/schedview/app/scheduler_dashboard/utils.py
@@ -10,12 +10,12 @@
 
 
 async def query_night_schedulers(reference_time_utc, efd="usdf_efd"):
-    """Query the EFD for the night schedulers
+    """Query the EFD for the night schedulers till reference time
 
     Parameters
     ----------
-    selected_time : `str`, 'float', or `astropy.time.Time`
-        The night to query, in YYYY-MM-DD format
+    reference_time_utc : `astropy.time.Time`
+        selected reference time in UTC
     efd : `str`
         The name of the EFD to query, usdf_efd or summit_efd
         Defaults to usdf_efd
@@ -25,27 +25,18 @@ async def query_night_schedulers(reference_time_utc, efd="usdf_efd"):
     schedulers : `list`
         A list of the schedulers for the night
     """
-
-    # reference_time_utc = Time(selected_time)
-    # print(f"reference_time_utc {reference_time_utc}")
-    # tz = timezone('Etc/GMT+12')
-    # reference_timestamp_utc_12 =
-    # Timestamp(reference_time_utc.to_datetime(timezone=tz))
-    # reference_date_utc_12 = reference_timestamp_utc_12.date()
-    # print(f"reference_date_utc_12 {reference_date_utc_12}")
-    # print(f"reference_timestamp_utc_12 {reference_timestamp_utc_12}")
-    # end_time = Time(str(reference_timestamp_utc_12))
-    # start_time = Time(Timestamp(str(reference_date_utc_12)))
-
+    # Get the night by converting the UTC time to timezone UTC-12
     tz = timezone("Etc/GMT+12")
     reference_time_utc_12 = Timestamp(reference_time_utc.to_datetime(timezone=tz))
     night = Time(str(reference_time_utc_12.date()))
+    # The offset by longitude moves to local solar time.
+    # The offset of 1 makes the night specificed refer to the
+    # local date corresponding to sunset.
     local_midnight = Time(night.mjd + 1 - Site("LSST").longitude / 360, format="mjd")
-    print(f"local midnight {local_midnight.to_datetime()}")
+    # Offset by half a jd to get the start of the night
     start_time = local_midnight - TimeDelta(0.5, format="jd")
+    # set the end time to the selected time
     end_time = reference_time_utc
-    print(f"star_time {start_time.to_datetime()}")
-    print(f"end_time {end_time}")
     efd_client = EfdClient(efd)
     topic = "lsst.sal.Scheduler.logevent_largeFileObjectAvailable"
     fields = ["url"]

From e4add12ef4adb7209b8afb135c0fc9555015df6c Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome002.sdf.slac.stanford.edu>
Date: Sun, 14 Apr 2024 06:37:40 -0700
Subject: [PATCH 07/21] allow selecting a telescope

---
 .../scheduler_dashboard.py                    | 32 +++++++++++++------
 schedview/app/scheduler_dashboard/utils.py    |  6 ++--
 2 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index b66a2cfc..e3b3781b 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -156,14 +156,14 @@ class Scheduler(param.Parameterized):
         default="",
         label="Scheduler pickle file",
         doc=scheduler_fname_doc,
-        precedence=2,
+        precedence=3,
     )
     widget_datetime = param.Date(
         default=date_bounds[0],
         label="Date and time (UTC)",
         doc=f"Select dates between {date_bounds[0]} and {date_bounds[1]}",
         bounds=date_bounds,
-        precedence=3,
+        precedence=4,
     )
     url_mjd = param.Number(default=None)
     widget_tier = param.Selector(
@@ -171,7 +171,7 @@ class Scheduler(param.Parameterized):
         objects=[""],
         label="Tier",
         doc="The label for the first index into the CoreScheduler.survey_lists.",
-        precedence=4,
+        precedence=5,
     )
     survey_map = param.Selector(
         default="reward",
@@ -1476,17 +1476,21 @@ class USDFScheduler(Scheduler):
         default="",
         objects=[],
         doc=scheduler_fname_doc,
-        precedence=2,
+        precedence=3,
     )
 
     pickles_date = param.Date(
         default=datetime.now(), label="Pickles Date", doc="Select date to load pickles for", precedence=1
     )
 
+    telescope = param.Selector(
+        default=None, objects={"All": None, "Auxtel": 1, "Main": 2}, doc="Source Telescope", precedence=2
+    )
+
     def __init__(self):
         super().__init__()
 
-    async def query_schedulers(self, selected_time):
+    async def query_schedulers(self, selected_time, selected_tel):
         selected_time = Time(
             Timestamp(
                 selected_time,
@@ -1494,7 +1498,7 @@ async def query_schedulers(self, selected_time):
             )
         )
         self.show_loading_indicator = True
-        scheduler_urls = await query_night_schedulers(selected_time)
+        scheduler_urls = await query_night_schedulers(selected_time, selected_tel)
         self.show_loading_indicator = False
         return scheduler_urls
 
@@ -1576,16 +1580,24 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
     # Load pickles from USDF S3 bucket
     elif from_usdf:
         scheduler = USDFScheduler()
-        data_loading_parameters = ["scheduler_fname", "pickles_date", "widget_datetime", "widget_tier"]
+        data_loading_parameters = [
+            "scheduler_fname",
+            "pickles_date",
+            "telescope",
+            "widget_datetime",
+            "widget_tier",
+        ]
         data_loading_widgets = {
             "pickles_date": pn.widgets.DatetimePicker,
             "widget_datetime": pn.widgets.DatetimePicker,
         }
 
-        @pn.depends(selected_time=scheduler.param.pickles_date, watch=True)
-        async def get_scheduler_list(selected_time):
+        @pn.depends(
+            selected_time=scheduler.param.pickles_date, selected_tel=scheduler.param.telescope, watch=True
+        )
+        async def get_scheduler_list(selected_time, selected_tel):
             os.environ["LSST_DISABLE_BUCKET_VALIDATION"] = "1"
-            schedulers = await scheduler.query_schedulers(selected_time)
+            schedulers = await scheduler.query_schedulers(selected_time, selected_tel)
             scheduler.param["scheduler_fname"].objects = schedulers
 
     # Restrict files to data_directory.
diff --git a/schedview/app/scheduler_dashboard/utils.py b/schedview/app/scheduler_dashboard/utils.py
index 259400e8..f5e26be0 100644
--- a/schedview/app/scheduler_dashboard/utils.py
+++ b/schedview/app/scheduler_dashboard/utils.py
@@ -9,7 +9,7 @@
 LOCAL_ROOT_URI = {"usdf": "s3://rubin:", "summit": "https://s3.cp.lsst.org/"}
 
 
-async def query_night_schedulers(reference_time_utc, efd="usdf_efd"):
+async def query_night_schedulers(reference_time_utc, selected_telescope=None, efd="usdf_efd"):
     """Query the EFD for the night schedulers till reference time
 
     Parameters
@@ -40,7 +40,9 @@ async def query_night_schedulers(reference_time_utc, efd="usdf_efd"):
     efd_client = EfdClient(efd)
     topic = "lsst.sal.Scheduler.logevent_largeFileObjectAvailable"
     fields = ["url"]
-    scheduler_urls = await efd_client.select_time_series(topic, fields, start_time, end_time)
+    scheduler_urls = await efd_client.select_time_series(
+        topic, fields, start_time, end_time, index=selected_telescope
+    )
     if not scheduler_urls.empty:
         scheduler_urls.index.name = "time"
         scheduler_urls = scheduler_urls.reset_index()

From 667f04ab586adb8cd5c7dd61416ca1145568c6c9 Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome002.sdf.slac.stanford.edu>
Date: Wed, 17 Apr 2024 22:47:44 -0700
Subject: [PATCH 08/21] adjust layout

---
 .../scheduler_dashboard/scheduler_dashboard.py    | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index e3b3781b..230036c7 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -1660,17 +1660,18 @@ def handle_reload_pickle(event):
         sizing_mode="stretch_width",
         styles={"background": "#048b8c"},
     )
-    # Parameter inputs (pickle, widget_datetime, tier).
-    sched_app[8:36, 0:21] = pn.Param(
+    # Parameter inputs (pickle, widget_datetime, tier)
+    # as well as pickles date and telescope when running in USDF
+    sched_app[8:42, 0:21] = pn.Param(
         scheduler,
         parameters=data_loading_parameters,
         widgets=data_loading_widgets,
         name="Select pickle file, date and tier.",
     )
     # Reset button.
-    sched_app[36:42, 3:15] = pn.Row(reset_button)
+    sched_app[42:48, 3:15] = pn.Row(reset_button)
     # Survey rewards table and header.
-    sched_app[8:42, 21:67] = pn.Row(
+    sched_app[8:48, 21:67] = pn.Row(
         pn.Spacer(width=10),
         pn.Column(
             pn.Spacer(height=10),
@@ -1685,7 +1686,7 @@ def handle_reload_pickle(event):
         # sizing_mode="stretch_height",
     )
     # Reward table and header.
-    sched_app[42:87, 0:67] = pn.Row(
+    sched_app[48:90, 0:67] = pn.Row(
         pn.Spacer(width=10),
         pn.Column(
             pn.Spacer(height=10),
@@ -1707,7 +1708,7 @@ def handle_reload_pickle(event):
         pn.param.ParamMethod(scheduler.publish_sky_map, loading_indicator=True),
     )
     # Map display parameters (map, nside, color palette).
-    sched_app[74:87, 67:100] = pn.Param(
+    sched_app[74:90, 67:100] = pn.Param(
         scheduler,
         widgets={
             "survey_map": {"type": pn.widgets.Select, "width": 250},
@@ -1719,7 +1720,7 @@ def handle_reload_pickle(event):
         default_layout=pn.Row,
     )
     # Debugging collapsable card.
-    sched_app[87:100, :] = pn.Card(
+    sched_app[90:100, :] = pn.Card(
         scheduler._debugging_messages,
         header=pn.pane.Str("Debugging", stylesheets=[h2_stylesheet]),
         header_color="white",

From 1268e4fece917363f008ff713ab795bca97d6122 Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome001.sdf.slac.stanford.edu>
Date: Sun, 21 Apr 2024 21:15:43 -0700
Subject: [PATCH 09/21] fix layout in USDF mode

---
 .../app/scheduler_dashboard/scheduler_dashboard.py     | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index 230036c7..005f5b53 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -799,7 +799,7 @@ def create_summary_widget(self):
             selectable=1,
             hidden_columns=["tier", "survey_url"],
             sizing_mode="stretch_width",
-            height=220,
+            height=310,
         )
         self.summary_widget = summary_widget
         self._debugging_message = "Finished making summary widget."
@@ -1680,13 +1680,11 @@ def handle_reload_pickle(event):
                 styles={"background": "#048b8c"},
             ),
             pn.param.ParamMethod(scheduler.publish_summary_widget, loading_indicator=True),
-            # scroll=True
         ),
         pn.Spacer(width=10),
-        # sizing_mode="stretch_height",
     )
     # Reward table and header.
-    sched_app[48:90, 0:67] = pn.Row(
+    sched_app[48:93, 0:67] = pn.Row(
         pn.Spacer(width=10),
         pn.Column(
             pn.Spacer(height=10),
@@ -1708,7 +1706,7 @@ def handle_reload_pickle(event):
         pn.param.ParamMethod(scheduler.publish_sky_map, loading_indicator=True),
     )
     # Map display parameters (map, nside, color palette).
-    sched_app[74:90, 67:100] = pn.Param(
+    sched_app[74:93, 67:100] = pn.Param(
         scheduler,
         widgets={
             "survey_map": {"type": pn.widgets.Select, "width": 250},
@@ -1720,7 +1718,7 @@ def handle_reload_pickle(event):
         default_layout=pn.Row,
     )
     # Debugging collapsable card.
-    sched_app[90:100, :] = pn.Card(
+    sched_app[93:100, :] = pn.Card(
         scheduler._debugging_messages,
         header=pn.pane.Str("Debugging", stylesheets=[h2_stylesheet]),
         header_color="white",

From fcd3275061d3eee0e06856e0314351282473de93 Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome001.sdf.slac.stanford.edu>
Date: Mon, 22 Apr 2024 00:16:44 -0700
Subject: [PATCH 10/21] Add an empty option to retrieved snapshots, added
 notifications and debugging messages

---
 .../scheduler_dashboard.py                    | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index 005f5b53..4d4a4d5a 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -1480,11 +1480,11 @@ class USDFScheduler(Scheduler):
     )
 
     pickles_date = param.Date(
-        default=datetime.now(), label="Pickles Date", doc="Select date to load pickles for", precedence=1
+        default=datetime.now(), label="Snapshot Date", doc="Select date to load pickles for", precedence=1
     )
 
     telescope = param.Selector(
-        default=None, objects={"All": None, "Auxtel": 1, "Main": 2}, doc="Source Telescope", precedence=2
+        default=None, objects={"All": None, "Main": 1, "Auxtel": 2}, doc="Source Telescope", precedence=2
     )
 
     def __init__(self):
@@ -1498,7 +1498,11 @@ async def query_schedulers(self, selected_time, selected_tel):
             )
         )
         self.show_loading_indicator = True
+        self._debugging_message = "Starting retrieving snapshots"
+        self.logger.debug("Starting retrieving snapshots")
         scheduler_urls = await query_night_schedulers(selected_time, selected_tel)
+        self.logger.debug("Finished retrieving snapshots")
+        self._debugging_message = "Finished retrieving snapshots"
         self.show_loading_indicator = False
         return scheduler_urls
 
@@ -1596,9 +1600,18 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
             selected_time=scheduler.param.pickles_date, selected_tel=scheduler.param.telescope, watch=True
         )
         async def get_scheduler_list(selected_time, selected_tel):
+            pn.state.notifications.clear()
+            pn.state.notifications.info("Loading snapshots...")
             os.environ["LSST_DISABLE_BUCKET_VALIDATION"] = "1"
-            schedulers = await scheduler.query_schedulers(selected_time, selected_tel)
+            # add an empty option to be seleced upon loading snapshot list
+            schedulers = [""]
+            schedulers[1:] = await scheduler.query_schedulers(selected_time, selected_tel)
             scheduler.param["scheduler_fname"].objects = schedulers
+            scheduler.clear_dashboard()
+            if len(schedulers) > 1:
+                pn.state.notifications.success("Snapshots loaded!!")
+            else:
+                pn.state.notifications.info("No snapshots found for selected night!!", duration=0)
 
     # Restrict files to data_directory.
     else:

From 8d20160b311a52f2403d9fe8f85032b057de49e1 Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome001.sdf.slac.stanford.edu>
Date: Mon, 22 Apr 2024 18:35:30 -0700
Subject: [PATCH 11/21] make gridspec definition changes according to dashboard
 mode as data loading paramaters change

---
 .../scheduler_dashboard.py                    | 22 +++++++++++--------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index 4d4a4d5a..d15fa810 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -223,6 +223,7 @@ class Scheduler(param.Parameterized):
     _display_reward = False
     _display_dashboard_data = False
     _do_not_trigger_update = True
+    _summary_widget_height = 220
 
     def __init__(self, **params):
         super().__init__(**params)
@@ -572,7 +573,7 @@ def _update_color_palette(self):
     # ------------------------------------------------------- Internal workings
 
     def clear_dashboard(self):
-        """Clear the dashboard for a new pickle or a new date."""
+        """Clear the dashboard for a new pickle or a new datesummary_widget."""
         self._debugging_message = "Starting to clear dashboard."
 
         self.summary_widget = None
@@ -799,7 +800,7 @@ def create_summary_widget(self):
             selectable=1,
             hidden_columns=["tier", "survey_url"],
             sizing_mode="stretch_width",
-            height=310,
+            height=self._summary_widget_height,
         )
         self.summary_widget = summary_widget
         self._debugging_message = "Finished making summary widget."
@@ -1487,6 +1488,8 @@ class USDFScheduler(Scheduler):
         default=None, objects={"All": None, "Main": 1, "Auxtel": 2}, doc="Source Telescope", precedence=2
     )
 
+    _summary_widget_height = 310
+
     def __init__(self):
         super().__init__()
 
@@ -1551,7 +1554,7 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
     scheduler = None
     data_loading_widgets = {}
     data_loading_parameters = ["scheduler_fname", "widget_datetime", "widget_tier"]
-
+    data_params_grid_height = 30
     # Accept pickle files from url or any path.
     if from_urls:
         scheduler = Scheduler()
@@ -1595,6 +1598,7 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
             "pickles_date": pn.widgets.DatetimePicker,
             "widget_datetime": pn.widgets.DatetimePicker,
         }
+        data_params_grid_height = 42
 
         @pn.depends(
             selected_time=scheduler.param.pickles_date, selected_tel=scheduler.param.telescope, watch=True
@@ -1675,16 +1679,16 @@ def handle_reload_pickle(event):
     )
     # Parameter inputs (pickle, widget_datetime, tier)
     # as well as pickles date and telescope when running in USDF
-    sched_app[8:42, 0:21] = pn.Param(
+    sched_app[8:data_params_grid_height, 0:21] = pn.Param(
         scheduler,
         parameters=data_loading_parameters,
         widgets=data_loading_widgets,
         name="Select pickle file, date and tier.",
     )
     # Reset button.
-    sched_app[42:48, 3:15] = pn.Row(reset_button)
+    sched_app[data_params_grid_height : data_params_grid_height + 6, 3:15] = pn.Row(reset_button)
     # Survey rewards table and header.
-    sched_app[8:48, 21:67] = pn.Row(
+    sched_app[8 : data_params_grid_height + 6, 21:67] = pn.Row(
         pn.Spacer(width=10),
         pn.Column(
             pn.Spacer(height=10),
@@ -1697,7 +1701,7 @@ def handle_reload_pickle(event):
         pn.Spacer(width=10),
     )
     # Reward table and header.
-    sched_app[48:93, 0:67] = pn.Row(
+    sched_app[data_params_grid_height + 6 : data_params_grid_height + 45, 0:67] = pn.Row(
         pn.Spacer(width=10),
         pn.Column(
             pn.Spacer(height=10),
@@ -1719,7 +1723,7 @@ def handle_reload_pickle(event):
         pn.param.ParamMethod(scheduler.publish_sky_map, loading_indicator=True),
     )
     # Map display parameters (map, nside, color palette).
-    sched_app[74:93, 67:100] = pn.Param(
+    sched_app[74 : data_params_grid_height + 45, 67:100] = pn.Param(
         scheduler,
         widgets={
             "survey_map": {"type": pn.widgets.Select, "width": 250},
@@ -1731,7 +1735,7 @@ def handle_reload_pickle(event):
         default_layout=pn.Row,
     )
     # Debugging collapsable card.
-    sched_app[93:100, :] = pn.Card(
+    sched_app[data_params_grid_height + 45 : data_params_grid_height + 52, :] = pn.Card(
         scheduler._debugging_messages,
         header=pn.pane.Str("Debugging", stylesheets=[h2_stylesheet]),
         header_color="white",

From 20f78650e1325e4e2e38af9753ecec673e9f5b8f Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome002.sdf.slac.stanford.edu>
Date: Mon, 22 Apr 2024 18:58:58 -0700
Subject: [PATCH 12/21] fix position of map display parameters in urls/data
 directory modes

---
 schedview/app/scheduler_dashboard/scheduler_dashboard.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index d15fa810..c6902e4c 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -1714,7 +1714,7 @@ def handle_reload_pickle(event):
         pn.Spacer(width=10),
     )
     # Map display and header.
-    sched_app[8:67, 67:100] = pn.Column(
+    sched_app[8 : data_params_grid_height + 25, 67:100] = pn.Column(
         pn.Spacer(height=10),
         pn.Row(
             scheduler.map_title,
@@ -1723,7 +1723,7 @@ def handle_reload_pickle(event):
         pn.param.ParamMethod(scheduler.publish_sky_map, loading_indicator=True),
     )
     # Map display parameters (map, nside, color palette).
-    sched_app[74 : data_params_grid_height + 45, 67:100] = pn.Param(
+    sched_app[data_params_grid_height + 32 : data_params_grid_height + 45, 67:100] = pn.Param(
         scheduler,
         widgets={
             "survey_map": {"type": pn.widgets.Select, "width": 250},

From 9aa5fc06c6c0582fe4ca862075de197fd522e0af Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome001.sdf.slac.stanford.edu>
Date: Mon, 22 Apr 2024 20:23:29 -0700
Subject: [PATCH 13/21] add more comments

---
 .../scheduler_dashboard/scheduler_dashboard.py   | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index c6902e4c..4870ab83 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -1553,7 +1553,12 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
 
     scheduler = None
     data_loading_widgets = {}
+    # data loading parameters in both restricted and URL modes
     data_loading_parameters = ["scheduler_fname", "widget_datetime", "widget_tier"]
+    # set the data loading parameter section height in both
+    # restricted and URL modes
+    # this will be used to adjust the layout of other sections
+    # in the grid
     data_params_grid_height = 30
     # Accept pickle files from url or any path.
     if from_urls:
@@ -1576,10 +1581,10 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
                     "url_mjd": "mjd",
                 },
             )
-
+        # set specific widget props for data loading parameters
+        # in URL and restricted modes
         data_loading_widgets = {
             "scheduler_fname": {
-                # "widget_type": pn.widgets.TextInput,
                 "placeholder": "filepath or URL of pickle",
             },
             "widget_datetime": pn.widgets.DatetimePicker,
@@ -1587,6 +1592,7 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
     # Load pickles from USDF S3 bucket
     elif from_usdf:
         scheduler = USDFScheduler()
+        # data loading parameters in USDF mode
         data_loading_parameters = [
             "scheduler_fname",
             "pickles_date",
@@ -1594,10 +1600,13 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
             "widget_datetime",
             "widget_tier",
         ]
+        # set specific widget props for data loading parameters
+        # in USDF mode
         data_loading_widgets = {
             "pickles_date": pn.widgets.DatetimePicker,
             "widget_datetime": pn.widgets.DatetimePicker,
         }
+        # set the data loading parameter section height in USDF mode
         data_params_grid_height = 42
 
         @pn.depends(
@@ -1607,7 +1616,8 @@ async def get_scheduler_list(selected_time, selected_tel):
             pn.state.notifications.clear()
             pn.state.notifications.info("Loading snapshots...")
             os.environ["LSST_DISABLE_BUCKET_VALIDATION"] = "1"
-            # add an empty option to be seleced upon loading snapshot list
+            # add an empty option at index 0 to be the default
+            # selection upon loading snapshot list
             schedulers = [""]
             schedulers[1:] = await scheduler.query_schedulers(selected_time, selected_tel)
             scheduler.param["scheduler_fname"].objects = schedulers

From a63e576be2fd769c4cfa982971eb09cbc10ab91f Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome001.sdf.slac.stanford.edu>
Date: Mon, 22 Apr 2024 20:30:11 -0700
Subject: [PATCH 14/21] add more comments

---
 schedview/app/scheduler_dashboard/scheduler_dashboard.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index 4870ab83..4380c828 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -573,7 +573,7 @@ def _update_color_palette(self):
     # ------------------------------------------------------- Internal workings
 
     def clear_dashboard(self):
-        """Clear the dashboard for a new pickle or a new datesummary_widget."""
+        """Clear the dashboard for a new pickle or a new date"""
         self._debugging_message = "Starting to clear dashboard."
 
         self.summary_widget = None
@@ -1469,7 +1469,6 @@ class USDFScheduler(Scheduler):
     scheduler dashboard.
     """
 
-    # Param parameters that are modifiable by user actions.
     scheduler_fname_doc = """Recent pickles from USDF
     """
 
@@ -1494,6 +1493,9 @@ def __init__(self):
         super().__init__()
 
     async def query_schedulers(self, selected_time, selected_tel):
+        """Query snapshots that have a timestamp between the start of the
+        night and selected datetime and generated by selected telescope
+        """
         selected_time = Time(
             Timestamp(
                 selected_time,

From fe11e260e6f2fef8969b67b616fbbfb53e5372a3 Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome001.sdf.slac.stanford.edu>
Date: Mon, 22 Apr 2024 20:35:13 -0700
Subject: [PATCH 15/21] add more comments

---
 schedview/app/scheduler_dashboard/utils.py | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/schedview/app/scheduler_dashboard/utils.py b/schedview/app/scheduler_dashboard/utils.py
index f5e26be0..5be2d973 100644
--- a/schedview/app/scheduler_dashboard/utils.py
+++ b/schedview/app/scheduler_dashboard/utils.py
@@ -44,10 +44,13 @@ async def query_night_schedulers(reference_time_utc, selected_telescope=None, ef
         topic, fields, start_time, end_time, index=selected_telescope
     )
     if not scheduler_urls.empty:
+        # index data by time
         scheduler_urls.index.name = "time"
         scheduler_urls = scheduler_urls.reset_index()
+        # reverse schedulers order to show most recent one first
         scheduler_urls = scheduler_urls.sort_index(ascending=False)
         scheduler_urls["url"] = scheduler_urls["url"].apply(lambda x: localize_scheduler_url(x))
+        # return only URLs
         return scheduler_urls["url"]
     return []
 
@@ -116,16 +119,3 @@ def mock_schedulers_df():
     df = df.sort_index(ascending=False)
     df["url"] = df["url"].apply(lambda x: localize_scheduler_url(x))
     return df["url"]
-
-
-async def get_top_n_schedulers(efd="usdf_efd"):
-    sync_client = EfdClient(efd, db_name="efd")
-    topic = "lsst.sal.Scheduler.logevent_largeFileObjectAvailable"
-    fields = ["url"]
-    scheduler_urls = await sync_client.select_top_n(topic, fields, 10)
-    scheduler_urls.index.name = "time"
-    scheduler_urls = scheduler_urls.reset_index()
-
-    scheduler_urls = scheduler_urls.sort_index(ascending=False)
-    scheduler_urls["url"] = scheduler_urls["url"].apply(lambda x: localize_scheduler_url(x))
-    return scheduler_urls["url"]

From 4ee8ffa134e20eab9133f3344349f54ac1194ba3 Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome001.sdf.slac.stanford.edu>
Date: Mon, 22 Apr 2024 21:33:45 -0700
Subject: [PATCH 16/21] remove the assert as it fails while running tests

---
 schedview/app/scheduler_dashboard/scheduler_dashboard.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index 4380c828..06bf01c1 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -614,7 +614,6 @@ def read_scheduler(self):
             os.environ["LSST_DISABLE_BUCKET_VALIDATION"] = "1"
             scheduler_resource_path = ResourcePath(self.scheduler_fname)
             scheduler_resource_path.use_threads = False
-            assert not scheduler_resource_path._environ_use_threads
             with scheduler_resource_path.as_local() as local_scheduler_resource:
                 (scheduler, conditions) = schedview.collect.scheduler_pickle.read_scheduler(
                     local_scheduler_resource.ospath

From 5ea51eac322aca47f338c964af06e189301197d5 Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome001.sdf.slac.stanford.edu>
Date: Mon, 22 Apr 2024 22:11:29 -0700
Subject: [PATCH 17/21] restore removed code to cater for reading sample
 scheduler objects

---
 schedview/collect/scheduler_pickle.py | 33 ++++++++++++++++++++++++---
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/schedview/collect/scheduler_pickle.py b/schedview/collect/scheduler_pickle.py
index 86f11679..7304792f 100644
--- a/schedview/collect/scheduler_pickle.py
+++ b/schedview/collect/scheduler_pickle.py
@@ -6,6 +6,10 @@
 import lzma
 import os
 import pickle
+import urllib
+import urllib.request
+from pathlib import Path
+from tempfile import TemporaryDirectory
 
 from lsst.resources import ResourcePath
 from rubin_scheduler.scheduler.model_observatory import ModelObservatory
@@ -77,10 +81,33 @@ def read_scheduler(file_name_or_url=None):
     conditions : `rubin_scheduler.scheduler.features.Conditions`
         An instance of a rubin_scheduler conditions object.
     """
+    if file_name_or_url is None:
+        file_name_or_url = PICKLE_FNAME
+
+    if file_name_or_url is None:
+        file_name_or_url = sample_pickle()
+
+    if Path(file_name_or_url).is_file():
+        scheduler_resource_path = ResourcePath(file_name_or_url)
+        with scheduler_resource_path.as_local() as local_scheduler_resource:
+            (scheduler, conditions) = read_local_scheduler_pickle(local_scheduler_resource.ospath)
+    else:
+        with TemporaryDirectory() as directory:
+            with urllib.request.urlopen(file_name_or_url) as url_io:
+                content = url_io.read()
+
+            # Infer a file name
+            parsed_url = urllib.parse.urlparse(file_name_or_url)
+            origin_path = Path(parsed_url.path)
+            origin_name = origin_path.name
+            name = origin_name if len(origin_name) > 0 else "scheduler.pickle"
+            path = Path(directory).joinpath(name)
+
+            with open(path, "wb") as file_io:
+                file_io.write(content)
+
+            scheduler, conditions = read_local_scheduler_pickle(str(path))
 
-    scheduler_resource_path = ResourcePath(file_name_or_url)
-    with scheduler_resource_path.as_local() as local_scheduler_resource:
-        (scheduler, conditions) = read_local_scheduler_pickle(local_scheduler_resource.ospath)
     return scheduler, conditions
 
 

From 6f19dc785a63a54b01a2328ccf354b75a8f56842 Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome002.sdf.slac.stanford.edu>
Date: Tue, 23 Apr 2024 15:50:41 -0700
Subject: [PATCH 18/21] fix the rewards table size in reestricted and url modes

---
 schedview/app/scheduler_dashboard/scheduler_dashboard.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index 06bf01c1..164fabe4 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -224,6 +224,7 @@ class Scheduler(param.Parameterized):
     _display_dashboard_data = False
     _do_not_trigger_update = True
     _summary_widget_height = 220
+    _reward_widget_height = 400
 
     def __init__(self, **params):
         super().__init__(**params)
@@ -972,7 +973,7 @@ def create_reward_widget(self):
             frozen_columns=["basis_function"],
             hidden_columns=["doc_url"],
             selectable=1,
-            height=400,
+            height=self._reward_widget_height,
             widths=widths,
         )
         self.reward_widget = reward_widget
@@ -1487,6 +1488,7 @@ class USDFScheduler(Scheduler):
     )
 
     _summary_widget_height = 310
+    _reward_widget_height = 350
 
     def __init__(self):
         super().__init__()

From 267cb14f1a8bf06f5d6612acfcfa11c9f2fbe110 Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome001.sdf.slac.stanford.edu>
Date: Tue, 23 Apr 2024 19:33:34 -0700
Subject: [PATCH 19/21] change USDF mode to LFA and change class names to
 reflect how it will be used in reality

---
 .../scheduler_dashboard.py                    | 44 +++++++++----------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index 164fabe4..b08f633a 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -71,7 +71,7 @@
 DEFAULT_COLOR_PALETTE = "Viridis256"
 DEFAULT_NSIDE = 16
 PACKAGE_DATA_DIR = importlib.resources.files("schedview.data").as_posix()
-USDF_DATA_DIR = "s3://rubin:"
+LFA_DATA_DIR = "s3://rubin:"
 
 
 pn.extension(
@@ -135,7 +135,7 @@ def url_formatter(dataframe_row, name_column, url_column):
             <i class="fa fa-link"></i></a>'
 
 
-class Scheduler(param.Parameterized):
+class SchedulerSnapshotDashboard(param.Parameterized):
     """A Parametrized container for parameters, data, and panel objects for the
     scheduler dashboard.
     """
@@ -1437,7 +1437,7 @@ def map_title(self):
         return self.map_title_pane
 
 
-class RestrictedFilesScheduler(Scheduler):
+class RestrictedSchedulerSnapshotDashboard(SchedulerSnapshotDashboard):
     """A Parametrized container for parameters, data, and panel objects for the
     scheduler dashboard.
     """
@@ -1464,12 +1464,12 @@ def __init__(self, data_dir=None):
             self.param["scheduler_fname"].update(path=f"{data_dir}/*scheduler*.p*")
 
 
-class USDFScheduler(Scheduler):
+class LFASchedulerSnapshotDashboard(SchedulerSnapshotDashboard):
     """A Parametrized container for parameters, data, and panel objects for the
     scheduler dashboard.
     """
 
-    scheduler_fname_doc = """Recent pickles from USDF
+    scheduler_fname_doc = """Recent pickles from LFA
     """
 
     scheduler_fname = param.Selector(
@@ -1540,7 +1540,7 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
 
     from_urls = False
     data_dir = None
-    from_usdf = False
+    from_lfa = False
 
     if "data_from_urls" in kwargs.keys():
         from_urls = kwargs["data_from_urls"]
@@ -1550,9 +1550,9 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
         data_dir = kwargs["data_dir"]
         del kwargs["data_dir"]
 
-    if "usdf" in kwargs.keys():
-        from_usdf = kwargs["usdf"]
-        del kwargs["usdf"]
+    if "lfa" in kwargs.keys():
+        from_lfa = kwargs["lfa"]
+        del kwargs["lfa"]
 
     scheduler = None
     data_loading_widgets = {}
@@ -1565,7 +1565,7 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
     data_params_grid_height = 30
     # Accept pickle files from url or any path.
     if from_urls:
-        scheduler = Scheduler()
+        scheduler = SchedulerSnapshotDashboard()
         # read pickle and time if provided to the function in a notebook
         # it will be overriden if the dashboard runs in an app
         if date_time is not None:
@@ -1592,10 +1592,10 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
             },
             "widget_datetime": pn.widgets.DatetimePicker,
         }
-    # Load pickles from USDF S3 bucket
-    elif from_usdf:
-        scheduler = USDFScheduler()
-        # data loading parameters in USDF mode
+    # Load pickles from S3 bucket
+    elif from_lfa:
+        scheduler = LFASchedulerSnapshotDashboard()
+        # data loading parameters in LFA mode
         data_loading_parameters = [
             "scheduler_fname",
             "pickles_date",
@@ -1604,12 +1604,12 @@ def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
             "widget_tier",
         ]
         # set specific widget props for data loading parameters
-        # in USDF mode
+        # in LFA mode
         data_loading_widgets = {
             "pickles_date": pn.widgets.DatetimePicker,
             "widget_datetime": pn.widgets.DatetimePicker,
         }
-        # set the data loading parameter section height in USDF mode
+        # set the data loading parameter section height in LFA mode
         data_params_grid_height = 42
 
         @pn.depends(
@@ -1632,7 +1632,7 @@ async def get_scheduler_list(selected_time, selected_tel):
 
     # Restrict files to data_directory.
     else:
-        scheduler = RestrictedFilesScheduler(data_dir=data_dir)
+        scheduler = RestrictedSchedulerSnapshotDashboard(data_dir=data_dir)
         data_loading_widgets = {
             "widget_datetime": pn.widgets.DatetimePicker,
         }
@@ -1691,7 +1691,7 @@ def handle_reload_pickle(event):
         styles={"background": "#048b8c"},
     )
     # Parameter inputs (pickle, widget_datetime, tier)
-    # as well as pickles date and telescope when running in USDF
+    # as well as pickles date and telescope when running in LFA
     sched_app[8:data_params_grid_height, 0:21] = pn.Param(
         scheduler,
         parameters=data_loading_parameters,
@@ -1765,7 +1765,7 @@ def parse_arguments():
     Parse commandline arguments to read data directory if provided
     """
     parser = argparse.ArgumentParser(description="On-the-fly Rubin Scheduler dashboard")
-    default_data_dir = f"{USDF_DATA_DIR}/*" if os.path.exists(USDF_DATA_DIR) else PACKAGE_DATA_DIR
+    default_data_dir = f"{LFA_DATA_DIR}/*" if os.path.exists(LFA_DATA_DIR) else PACKAGE_DATA_DIR
 
     parser.add_argument(
         "--data_dir",
@@ -1782,9 +1782,9 @@ def parse_arguments():
     )
 
     parser.add_argument(
-        "--usdf",
+        "--lfa",
         action="store_true",
-        help="Loads pickle files from the data directory in USDF",
+        help="Loads pickle files from S3 buckets in LFA",
     )
 
     args = parser.parse_args()
@@ -1792,7 +1792,7 @@ def parse_arguments():
     if len(glob(args.data_dir)) == 0 and not args.data_from_urls:
         args.data_dir = PACKAGE_DATA_DIR
 
-    if args.usdf and len(glob(USDF_DATA_DIR)) == 0:
+    if args.lfa and len(glob(LFA_DATA_DIR)) == 0:
         args.data_dir = PACKAGE_DATA_DIR
 
     scheduler_app_params = args.__dict__

From 45ff2d3da2f4df49d3d010160b2e7b6eaeb24410 Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome001.sdf.slac.stanford.edu>
Date: Tue, 23 Apr 2024 19:36:38 -0700
Subject: [PATCH 20/21] follow comments guidelines

---
 schedview/app/scheduler_dashboard/scheduler_dashboard.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/schedview/app/scheduler_dashboard/scheduler_dashboard.py b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
index b08f633a..729b611c 100644
--- a/schedview/app/scheduler_dashboard/scheduler_dashboard.py
+++ b/schedview/app/scheduler_dashboard/scheduler_dashboard.py
@@ -574,7 +574,7 @@ def _update_color_palette(self):
     # ------------------------------------------------------- Internal workings
 
     def clear_dashboard(self):
-        """Clear the dashboard for a new pickle or a new date"""
+        """Clear the dashboard for a new pickle or a new date."""
         self._debugging_message = "Starting to clear dashboard."
 
         self.summary_widget = None

From d59259e1cb416d4f9f5da90036720b7e3c1cd588 Mon Sep 17 00:00:00 2001
From: Eman Ali <eali@sdfrome002.sdf.slac.stanford.edu>
Date: Tue, 23 Apr 2024 21:20:40 -0700
Subject: [PATCH 21/21] change Scheduler class name to
 SchedulerSnapshotDashboard in tests

---
 tests/test_scheduler_dashboard.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_scheduler_dashboard.py b/tests/test_scheduler_dashboard.py
index 86ceec3f..59386183 100644
--- a/tests/test_scheduler_dashboard.py
+++ b/tests/test_scheduler_dashboard.py
@@ -21,7 +21,7 @@
 
 import schedview
 from schedview.app.scheduler_dashboard.scheduler_dashboard import (
-    Scheduler,
+    SchedulerSnapshotDashboard,
     get_sky_brightness_date_bounds,
     scheduler_app,
 )
@@ -65,7 +65,7 @@
 
 class TestSchedulerDashboard(unittest.TestCase):
     observatory = ModelObservatory(init_load_length=1)
-    scheduler = Scheduler()
+    scheduler = SchedulerSnapshotDashboard()
     scheduler.scheduler_fname = TEST_PICKLE
     scheduler._mjd = Time(Timestamp(TEST_DATE, tzinfo=ZoneInfo(DEFAULT_TIMEZONE))).mjd
 
@@ -83,7 +83,7 @@ def test_scheduler_app(self):
         bokeh.io.reset_output()
 
     def test_read_scheduler(self):
-        self.scheduler = Scheduler()
+        self.scheduler = SchedulerSnapshotDashboard()
         self.scheduler.scheduler_fname = TEST_PICKLE
         self.scheduler._mjd = Time(Timestamp(TEST_DATE, tzinfo=ZoneInfo(DEFAULT_TIMEZONE))).mjd
         self.scheduler.read_scheduler()