DM-47777: Add SOAR seeing scraping and plotting

python/lsst/rubintv/production/plotting/nightReportPlotBase.py

@@ -179,7 +179,11 @@ def createAndUpload(self, nightReport, metadata, ccdVisitTable):
         plt.close()
 
         self.s3Uploader.uploadNightReportData(
-            instrument="auxtel", dayObs=self.dayObs, filename=saveFile, plotGroup=self.plotGroup
+            instrument="auxtel",
+            dayObs=self.dayObs,
+            filename=saveFile,
+            plotGroup=self.plotGroup,
+            uploadAs=self.plotName,
         )
         # if things start failing later you don't want old plots sticking
         # around and getting re-uploaded as if they were new
@@ -247,7 +251,11 @@ def createAndUpload(self, tableData):
         plt.close()
 
         self.s3Uploader.uploadNightReportData(
-            instrument="startracker", dayObs=self.dayObs, filename=saveFile, plotGroup=self.plotGroup
+            instrument="startracker",
+            dayObs=self.dayObs,
+            filename=saveFile,
+            plotGroup=self.plotGroup,
+            uploadAs=self.plotName,
         )
         # if things start failing later you don't want old plots sticking
         # around and getting re-uploaded as if they were new

python/lsst/rubintv/production/rubinTv.py

@@ -1272,7 +1272,11 @@ def callback(self, expRecord, doCheckDay=True):
                 airMassPlotFile = os.path.join(self.locationConfig.nightReportPath, "airmass.png")
                 self.report.plotPerObjectAirMass(saveFig=airMassPlotFile)
                 self.s3Uploader.uploadNightReportData(
-                    instrument="auxtel", dayObs=self.dayObs, filename=airMassPlotFile, plotGroup="Coverage"
+                    instrument="auxtel",
+                    dayObs=self.dayObs,
+                    filename=airMassPlotFile,
+                    plotGroup="Coverage",
+                    uploadAs="airmass.png",
                 )
 
                 # the alt/az coverage polar plot
@@ -1283,6 +1287,7 @@ def callback(self, expRecord, doCheckDay=True):
                     dayObs=self.dayObs,
                     filename=altAzCoveragePlotFile,
                     plotGroup="Coverage",
+                    uploadsAs="alt-az.png",
                 )
 
                 # Add text items here
@@ -1302,6 +1307,7 @@ def callback(self, expRecord, doCheckDay=True):
                     instrument="auxtel",
                     dayObs=self.dayObs,
                     filename=jsonFilename,
+                    isMetadataFile=True,
                 )
 
                 self.log.info(f"Finished updating plots and table for {dataId}")

python/lsst/rubintv/production/soarSeeing.py

@@ -0,0 +1,78 @@
+# This file is part of rubintv_production.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+__all__ = [
+    "SoarUploader",
+]
+
+import logging
+import tempfile
+from time import sleep
+
+import matplotlib.pyplot as plt
+from astropy.time import Time
+
+from lsst.rubintv.production.uploaders import MultiUploader
+from lsst.rubintv.production.utils import getRubinTvInstrumentName, raiseIf
+from lsst.summit.extras.soarSeeing import SoarSeeingMonitor
+from lsst.summit.utils.utils import getCurrentDayObs_int
+
+
+class SoarUploader:
+    def __init__(self, doRaise: bool) -> None:
+        self.soar = SoarSeeingMonitor()
+        self.instrument = "LSSTComCam"  # change to LSSTCam when we switch instruments
+        self.doRaise = doRaise
+        self.figure = plt.figure(figsize=(18, 10))
+        self.uploader = MultiUploader()
+        self._lastUpdated = Time.now()
+        self.log = logging.getLogger(__name__)
+
+    def newData(self) -> bool:
+        return self._lastUpdated != self.soar.getMostRecentTimestamp()
+
+    def run(self) -> None:
+        while True:
+            dayObs = getCurrentDayObs_int()
+            try:
+                if not self.newData():
+                    sleep(15)
+                    continue
+
+                self._lastUpdated = self.soar.getMostRecentTimestamp()
+                self.figure = self.soar.plotSeeingForDayObs(dayObs, addMostRecentBox=True, fig=self.figure)
+                with tempfile.NamedTemporaryFile(suffix=".png") as f:
+                    self.log.info(f"Uploading SOAR seeing to night report for {dayObs}")
+                    self.figure.savefig(f.name)
+                    self.uploader.uploadNightReportData(
+                        instrument=getRubinTvInstrumentName(self.instrument),
+                        dayObs=dayObs,
+                        filename=f.name,
+                        uploadAs="SoarSeeingMonitor.png",
+                        plotGroup="Seeing",
+                    )
+                    self.log.info("Done")
+                    self.figure.clear()
+
+            except Exception as e:
+                logging.error(f"Error: {e}")
+                raiseIf(self.doRaise, e, self.log)

python/lsst/rubintv/production/uploaders.py

@@ -32,7 +32,7 @@
 from boto3.session import Session as S3_session
 from botocore.config import Config
 from botocore.exceptions import BotoCoreError, ClientError
-from typing_extensions import Optional, override
+from typing_extensions import override
 
 from lsst.summit.utils.utils import dayObsIntToString, getSite
 
@@ -194,32 +194,50 @@ def __init__(self):
 
     @abstractmethod
     def uploadNightReportData(
-        self, channel: str, dayObs: int, filename: str, plotGroup: Optional[str]
+        self,
+        instrument: str,
+        dayObs: int,
+        filename: str,
+        uploadAs: str,
+        plotGroup: str = None,
+        *,
+        isMetadataFile: bool = False,
     ) -> str:
-        """Upload night report type plot or json file to a night
-           report channel.
+        """Upload night report type plot or json file
+           to a night report channel.
 
         Parameters
         ----------
-        channel : `str`
-            The RubinTV channel to upload to.
-        observation_day : `int`
+        instrument : `str`
+            The instrument.
+        dayObs : `int`
             The dayObs.
         filename : `str`
             The full path and filename of the file to upload.
+        uploadAs : `str`
+            The name of the desination file. Only include the last part, i.e.
+            how it should appear when displayed, including the extension.
         plotGroup : `str`, optional
             The group to upload the plot to. The 'default' group is used if
             this is not specified. However, people are encouraged to supply
             groups for their plots, so the 'default' value is not put in the
             function signature to indicate this.
+        isMetadataFile : `bool`, optional
+            If the file is the md.json file, set this to True. Will ignore the
+            plotGroup and uploadAs parameters.
 
         Raises
         ------
         ValueError
             Raised if the specified channel is not in the list of existing
-            channels as specified in CHANNELS.
+            channels as specified in CHANNELS
         UploadError
-            Raised if uploading the file to the Bucket was not possible.
+            Raised if uploading the file to the Bucket was not possible
+
+        Returns
+        -------
+        uploadAs: `str``
+            Path and filename for the destination file in the bucket
         """
         raise NotImplementedError()
 
@@ -569,7 +587,10 @@ def uploadNightReportData(
         instrument: str,
         dayObs: int,
         filename: str,
+        uploadAs: str,
         plotGroup: str = None,
+        *,
+        isMetadataFile: bool = False,
     ) -> str:
         """Upload night report type plot or json file
            to a night report channel.
@@ -582,11 +603,17 @@ def uploadNightReportData(
             The dayObs.
         filename : `str`
             The full path and filename of the file to upload.
+        uploadAs : `str`
+            The name of the desination file. Only include the last part, i.e.
+            how it should appear when displayed, including the extension.
         plotGroup : `str`, optional
             The group to upload the plot to. The 'default' group is used if
             this is not specified. However, people are encouraged to supply
             groups for their plots, so the 'default' value is not put in the
             function signature to indicate this.
+        isMetadataFile : `bool`, optional
+            If the file is the md.json file, set this to True. Will ignore the
+            plotGroup and uploadAs parameters.
 
         Raises
         ------
@@ -610,29 +637,23 @@ def uploadNightReportData(
         if plotGroup is None:
             plotGroup = "default"
 
-        basename = os.path.basename(filename)
         dayObsStr = dayObsIntToString(dayObs)
+        baseName = f"{instrument}/{dayObsStr}/night_report"
 
-        if basename == "md.json":  # it's not a plot, so special case this one case
-            uploadAs = (
-                f"{instrument}/{dayObsStr}/night_report/{instrument}_night_report_{dayObsStr}_{basename}"
-            )
+        if isMetadataFile:
+            destName = f"{baseName}/{instrument}_night_report_{dayObsStr}_md.json"
         else:
-            # the plot filenames have the channel name saved into them in the
-            # form path/channelName-plotName.png, so remove the channel name
-            # and dash
-            plotName = basename.replace(instrument + "_night_reports" + "-", "")
-            plotFilename = f"{instrument}_night_report_{dayObsStr}_{plotGroup}_{plotName}"
-            uploadAs = f"{instrument}/{dayObsStr}/night_report/{plotGroup}/{plotFilename}"
+            plotFilename = f"{instrument}_night_report_{dayObsStr}_{plotGroup}_{uploadAs}"
+            destName = f"{baseName}/{plotGroup}/{plotFilename}"
 
         try:
-            self.upload(destinationFilename=uploadAs, sourceFilename=filename)
-            self._log.info(f"Uploaded {filename} to {uploadAs}")
+            self.upload(destinationFilename=destName, sourceFilename=filename)
+            self._log.info(f"Uploaded {filename} to {destName}")
         except Exception as ex:
-            self._log.exception(f"Failed to upload {filename} as {uploadAs} for {instrument} night report")
+            self._log.exception(f"Failed to upload {filename} as {destName} for {instrument} night report")
             raise ex
 
-        return uploadAs
+        return destName
 
     @override
     def upload(self, destinationFilename: str, sourceFilename: str) -> str:
@@ -801,63 +822,6 @@ def uploadPerSeqNumPlot(
 
         return blob
 
-    def uploadNightReportData(
-        self,
-        channel,
-        dayObs,
-        filename,
-        plotGroup="",
-    ):
-        """Upload night report type plot or json file to a night report channel
-
-        Parameters
-        ----------
-        channel : `str`
-            The RubinTV channel to upload to.
-        dayObsInt : `int`
-            The dayObs.
-        filename : `str`
-            The full path and filename of the file to upload.
-        plotGroup : `str`, optional
-            The group to upload the plot to. The 'default' group is used if
-            this is not specified. However, people are encouraged to supply
-            groups for their plots, so the 'default' value is not put in the
-            function signature to indicate this.
-
-        Raises
-        ------
-        ValueError
-            Raised if the specified channel is not in the list of existing
-            channels as specified in CHANNELS
-        """
-        if channel not in CHANNELS:
-            raise ValueError(f"Error: {channel} not in {CHANNELS}")
-
-        basename = os.path.basename(filename)  # deals with png vs jpeg
-
-        # the plot filenames have the channel name saved into them in the form
-        # path/channelName-plotName.png, so remove the channel name and dash
-        basename = basename.replace(channel + "-", "")
-        uploadAs = f"{channel}/{dayObs}/{plotGroup if plotGroup else 'default'}/{basename}"
-
-        blob = self.bucket.blob(uploadAs)
-        blob.cache_control = "no-store"
-
-        # retry strategy here is also gentle as these plots are routintely
-        # updated, so we'll get a new version soon enough.
-        timeout = 60  # default is 60s
-        deadline = 2.0
-        modified_retry = DEFAULT_RETRY.with_deadline(deadline)  # in seconds
-        modified_retry = modified_retry.with_delay(initial=0.5, multiplier=1.2, maximum=2)
-        try:
-            blob.upload_from_filename(filename, retry=modified_retry, timeout=timeout)
-            self.log.info(f"Uploaded {filename} to {uploadAs}")
-        except Exception as e:
-            self.log.warning(f"Failed to upload {uploadAs} to {channel} because {repr(e)}")
-            return None
-
-        return blob
-
     def googleUpload(
         self,
         channel,

scripts/summit/misc/runSoarSeeingScraper.py

@@ -0,0 +1,26 @@
+# This file is part of rubintv_production.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from lsst.summit.extras.soarSeeing import SoarDatabaseBuiler
+
+scraper = SoarDatabaseBuiler()
+
+scraper.run()

scripts/summit/misc/runSoarSeeingUploader.py

@@ -0,0 +1,27 @@
+# This file is part of rubintv_production.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from lsst.rubintv.production.soarSeeing import SoarUploader
+from lsst.rubintv.production.utils import getDoRaise
+
+uploader = SoarUploader(doRaise=getDoRaise())
+
+uploader.run()

tests/test_s3_uploader.py

@@ -86,6 +86,7 @@ def test_uploadNightReportData(self) -> None:
             dayObs=observationDay,
             filename=filename,
             plotGroup=plotGroup,
+            uploadAs="test_file_0001.txt",
         )
         self.is_correct_check_uploaded_file(uploadedFile, fileContent)