diff --git a/python/lsst/rubintv/production/plotting/nightReportPlotBase.py b/python/lsst/rubintv/production/plotting/nightReportPlotBase.py
index 410310345..ff73615b0 100644
--- a/python/lsst/rubintv/production/plotting/nightReportPlotBase.py
+++ b/python/lsst/rubintv/production/plotting/nightReportPlotBase.py
@@ -179,7 +179,11 @@ def createAndUpload(self, nightReport, metadata, ccdVisitTable):
plt.close()
self.s3Uploader.uploadNightReportData(
- instrument="auxtel", dayObs=self.dayObs, filename=saveFile, plotGroup=self.plotGroup
+ instrument="auxtel",
+ dayObs=self.dayObs,
+ filename=saveFile,
+ plotGroup=self.plotGroup,
+ uploadAs=self.plotName,
)
# if things start failing later you don't want old plots sticking
# around and getting re-uploaded as if they were new
@@ -247,7 +251,11 @@ def createAndUpload(self, tableData):
plt.close()
self.s3Uploader.uploadNightReportData(
- instrument="startracker", dayObs=self.dayObs, filename=saveFile, plotGroup=self.plotGroup
+ instrument="startracker",
+ dayObs=self.dayObs,
+ filename=saveFile,
+ plotGroup=self.plotGroup,
+ uploadAs=self.plotName,
)
# if things start failing later you don't want old plots sticking
# around and getting re-uploaded as if they were new
diff --git a/python/lsst/rubintv/production/rubinTv.py b/python/lsst/rubintv/production/rubinTv.py
index 798f3d56b..b7a244e68 100644
--- a/python/lsst/rubintv/production/rubinTv.py
+++ b/python/lsst/rubintv/production/rubinTv.py
@@ -1272,7 +1272,11 @@ def callback(self, expRecord, doCheckDay=True):
airMassPlotFile = os.path.join(self.locationConfig.nightReportPath, "airmass.png")
self.report.plotPerObjectAirMass(saveFig=airMassPlotFile)
self.s3Uploader.uploadNightReportData(
- instrument="auxtel", dayObs=self.dayObs, filename=airMassPlotFile, plotGroup="Coverage"
+ instrument="auxtel",
+ dayObs=self.dayObs,
+ filename=airMassPlotFile,
+ plotGroup="Coverage",
+ uploadAs="airmass.png",
)
# the alt/az coverage polar plot
@@ -1283,6 +1287,7 @@ def callback(self, expRecord, doCheckDay=True):
dayObs=self.dayObs,
filename=altAzCoveragePlotFile,
plotGroup="Coverage",
+ uploadsAs="alt-az.png",
)
# Add text items here
@@ -1302,6 +1307,7 @@ def callback(self, expRecord, doCheckDay=True):
instrument="auxtel",
dayObs=self.dayObs,
filename=jsonFilename,
+ isMetadataFile=True,
)
self.log.info(f"Finished updating plots and table for {dataId}")
diff --git a/python/lsst/rubintv/production/soarSeeing.py b/python/lsst/rubintv/production/soarSeeing.py
new file mode 100644
index 000000000..dbb9cff8a
--- /dev/null
+++ b/python/lsst/rubintv/production/soarSeeing.py
@@ -0,0 +1,78 @@
+# This file is part of rubintv_production.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+from __future__ import annotations
+
+__all__ = [
+ "SoarUploader",
+]
+
+import logging
+import tempfile
+from time import sleep
+
+import matplotlib.pyplot as plt
+from astropy.time import Time
+
+from lsst.rubintv.production.uploaders import MultiUploader
+from lsst.rubintv.production.utils import getRubinTvInstrumentName, raiseIf
+from lsst.summit.extras.soarSeeing import SoarSeeingMonitor
+from lsst.summit.utils.utils import getCurrentDayObs_int
+
+
+class SoarUploader:
+ def __init__(self, doRaise: bool) -> None:
+ self.soar = SoarSeeingMonitor()
+ self.instrument = "LSSTComCam" # change to LSSTCam when we switch instruments
+ self.doRaise = doRaise
+ self.figure = plt.figure(figsize=(18, 10))
+ self.uploader = MultiUploader()
+ self._lastUpdated = Time.now()
+ self.log = logging.getLogger(__name__)
+
+ def newData(self) -> bool:
+ return self._lastUpdated != self.soar.getMostRecentTimestamp()
+
+ def run(self) -> None:
+ while True:
+ dayObs = getCurrentDayObs_int()
+ try:
+ if not self.newData():
+ sleep(15)
+ continue
+
+ self._lastUpdated = self.soar.getMostRecentTimestamp()
+ self.figure = self.soar.plotSeeingForDayObs(dayObs, addMostRecentBox=True, fig=self.figure)
+ with tempfile.NamedTemporaryFile(suffix=".png") as f:
+ self.log.info(f"Uploading SOAR seeing to night report for {dayObs}")
+ self.figure.savefig(f.name)
+ self.uploader.uploadNightReportData(
+ instrument=getRubinTvInstrumentName(self.instrument),
+ dayObs=dayObs,
+ filename=f.name,
+ uploadAs="SoarSeeingMonitor.png",
+ plotGroup="Seeing",
+ )
+ self.log.info("Done")
+ self.figure.clear()
+
+ except Exception as e:
+ logging.error(f"Error: {e}")
+ raiseIf(self.doRaise, e, self.log)
diff --git a/python/lsst/rubintv/production/uploaders.py b/python/lsst/rubintv/production/uploaders.py
index 7ef84c28e..2e39a73b8 100644
--- a/python/lsst/rubintv/production/uploaders.py
+++ b/python/lsst/rubintv/production/uploaders.py
@@ -32,7 +32,7 @@
from boto3.session import Session as S3_session
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
-from typing_extensions import Optional, override
+from typing_extensions import override
from lsst.summit.utils.utils import dayObsIntToString, getSite
@@ -194,32 +194,50 @@ def __init__(self):
@abstractmethod
def uploadNightReportData(
- self, channel: str, dayObs: int, filename: str, plotGroup: Optional[str]
+ self,
+ instrument: str,
+ dayObs: int,
+ filename: str,
+ uploadAs: str,
+ plotGroup: str = None,
+ *,
+ isMetadataFile: bool = False,
) -> str:
- """Upload night report type plot or json file to a night
- report channel.
+ """Upload night report type plot or json file
+ to a night report channel.
Parameters
----------
- channel : `str`
- The RubinTV channel to upload to.
- observation_day : `int`
+ instrument : `str`
+ The instrument.
+ dayObs : `int`
The dayObs.
filename : `str`
The full path and filename of the file to upload.
+ uploadAs : `str`
+ The name of the desination file. Only include the last part, i.e.
+ how it should appear when displayed, including the extension.
plotGroup : `str`, optional
The group to upload the plot to. The 'default' group is used if
this is not specified. However, people are encouraged to supply
groups for their plots, so the 'default' value is not put in the
function signature to indicate this.
+ isMetadataFile : `bool`, optional
+ If the file is the md.json file, set this to True. Will ignore the
+ plotGroup and uploadAs parameters.
Raises
------
ValueError
Raised if the specified channel is not in the list of existing
- channels as specified in CHANNELS.
+ channels as specified in CHANNELS
UploadError
- Raised if uploading the file to the Bucket was not possible.
+ Raised if uploading the file to the Bucket was not possible
+
+ Returns
+ -------
+ uploadAs: `str``
+ Path and filename for the destination file in the bucket
"""
raise NotImplementedError()
@@ -569,7 +587,10 @@ def uploadNightReportData(
instrument: str,
dayObs: int,
filename: str,
+ uploadAs: str,
plotGroup: str = None,
+ *,
+ isMetadataFile: bool = False,
) -> str:
"""Upload night report type plot or json file
to a night report channel.
@@ -582,11 +603,17 @@ def uploadNightReportData(
The dayObs.
filename : `str`
The full path and filename of the file to upload.
+ uploadAs : `str`
+ The name of the desination file. Only include the last part, i.e.
+ how it should appear when displayed, including the extension.
plotGroup : `str`, optional
The group to upload the plot to. The 'default' group is used if
this is not specified. However, people are encouraged to supply
groups for their plots, so the 'default' value is not put in the
function signature to indicate this.
+ isMetadataFile : `bool`, optional
+ If the file is the md.json file, set this to True. Will ignore the
+ plotGroup and uploadAs parameters.
Raises
------
@@ -610,29 +637,23 @@ def uploadNightReportData(
if plotGroup is None:
plotGroup = "default"
- basename = os.path.basename(filename)
dayObsStr = dayObsIntToString(dayObs)
+ baseName = f"{instrument}/{dayObsStr}/night_report"
- if basename == "md.json": # it's not a plot, so special case this one case
- uploadAs = (
- f"{instrument}/{dayObsStr}/night_report/{instrument}_night_report_{dayObsStr}_{basename}"
- )
+ if isMetadataFile:
+ destName = f"{baseName}/{instrument}_night_report_{dayObsStr}_md.json"
else:
- # the plot filenames have the channel name saved into them in the
- # form path/channelName-plotName.png, so remove the channel name
- # and dash
- plotName = basename.replace(instrument + "_night_reports" + "-", "")
- plotFilename = f"{instrument}_night_report_{dayObsStr}_{plotGroup}_{plotName}"
- uploadAs = f"{instrument}/{dayObsStr}/night_report/{plotGroup}/{plotFilename}"
+ plotFilename = f"{instrument}_night_report_{dayObsStr}_{plotGroup}_{uploadAs}"
+ destName = f"{baseName}/{plotGroup}/{plotFilename}"
try:
- self.upload(destinationFilename=uploadAs, sourceFilename=filename)
- self._log.info(f"Uploaded {filename} to {uploadAs}")
+ self.upload(destinationFilename=destName, sourceFilename=filename)
+ self._log.info(f"Uploaded {filename} to {destName}")
except Exception as ex:
- self._log.exception(f"Failed to upload {filename} as {uploadAs} for {instrument} night report")
+ self._log.exception(f"Failed to upload {filename} as {destName} for {instrument} night report")
raise ex
- return uploadAs
+ return destName
@override
def upload(self, destinationFilename: str, sourceFilename: str) -> str:
@@ -801,63 +822,6 @@ def uploadPerSeqNumPlot(
return blob
- def uploadNightReportData(
- self,
- channel,
- dayObs,
- filename,
- plotGroup="",
- ):
- """Upload night report type plot or json file to a night report channel
-
- Parameters
- ----------
- channel : `str`
- The RubinTV channel to upload to.
- dayObsInt : `int`
- The dayObs.
- filename : `str`
- The full path and filename of the file to upload.
- plotGroup : `str`, optional
- The group to upload the plot to. The 'default' group is used if
- this is not specified. However, people are encouraged to supply
- groups for their plots, so the 'default' value is not put in the
- function signature to indicate this.
-
- Raises
- ------
- ValueError
- Raised if the specified channel is not in the list of existing
- channels as specified in CHANNELS
- """
- if channel not in CHANNELS:
- raise ValueError(f"Error: {channel} not in {CHANNELS}")
-
- basename = os.path.basename(filename) # deals with png vs jpeg
-
- # the plot filenames have the channel name saved into them in the form
- # path/channelName-plotName.png, so remove the channel name and dash
- basename = basename.replace(channel + "-", "")
- uploadAs = f"{channel}/{dayObs}/{plotGroup if plotGroup else 'default'}/{basename}"
-
- blob = self.bucket.blob(uploadAs)
- blob.cache_control = "no-store"
-
- # retry strategy here is also gentle as these plots are routintely
- # updated, so we'll get a new version soon enough.
- timeout = 60 # default is 60s
- deadline = 2.0
- modified_retry = DEFAULT_RETRY.with_deadline(deadline) # in seconds
- modified_retry = modified_retry.with_delay(initial=0.5, multiplier=1.2, maximum=2)
- try:
- blob.upload_from_filename(filename, retry=modified_retry, timeout=timeout)
- self.log.info(f"Uploaded {filename} to {uploadAs}")
- except Exception as e:
- self.log.warning(f"Failed to upload {uploadAs} to {channel} because {repr(e)}")
- return None
-
- return blob
-
def googleUpload(
self,
channel,
diff --git a/scripts/summit/misc/runSoarSeeingScraper.py b/scripts/summit/misc/runSoarSeeingScraper.py
new file mode 100644
index 000000000..22166188b
--- /dev/null
+++ b/scripts/summit/misc/runSoarSeeingScraper.py
@@ -0,0 +1,26 @@
+# This file is part of rubintv_production.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+from lsst.summit.extras.soarSeeing import SoarDatabaseBuiler
+
+scraper = SoarDatabaseBuiler()
+
+scraper.run()
diff --git a/scripts/summit/misc/runSoarSeeingUploader.py b/scripts/summit/misc/runSoarSeeingUploader.py
new file mode 100644
index 000000000..62088c6e9
--- /dev/null
+++ b/scripts/summit/misc/runSoarSeeingUploader.py
@@ -0,0 +1,27 @@
+# This file is part of rubintv_production.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+from lsst.rubintv.production.soarSeeing import SoarUploader
+from lsst.rubintv.production.utils import getDoRaise
+
+uploader = SoarUploader(doRaise=getDoRaise())
+
+uploader.run()
diff --git a/tests/test_s3_uploader.py b/tests/test_s3_uploader.py
index 8d813ccdd..acf108ac1 100644
--- a/tests/test_s3_uploader.py
+++ b/tests/test_s3_uploader.py
@@ -86,6 +86,7 @@ def test_uploadNightReportData(self) -> None:
dayObs=observationDay,
filename=filename,
plotGroup=plotGroup,
+ uploadAs="test_file_0001.txt",
)
self.is_correct_check_uploaded_file(uploadedFile, fileContent)