From a6229f7f43e6fd61b59b2e49a0b2e6d4c4daa8c0 Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Sun, 1 Dec 2024 19:34:16 -0800 Subject: [PATCH 1/6] Add SOAR seeing scraping and plotting --- python/lsst/rubintv/production/soarSeeing.py | 74 ++++++++++++++++++++ scripts/summit/misc/runSoarSeeingScraper.py | 26 +++++++ scripts/summit/misc/runSoarSeeingUploader.py | 27 +++++++ 3 files changed, 127 insertions(+) create mode 100644 python/lsst/rubintv/production/soarSeeing.py create mode 100644 scripts/summit/misc/runSoarSeeingScraper.py create mode 100644 scripts/summit/misc/runSoarSeeingUploader.py diff --git a/python/lsst/rubintv/production/soarSeeing.py b/python/lsst/rubintv/production/soarSeeing.py new file mode 100644 index 000000000..ed438329e --- /dev/null +++ b/python/lsst/rubintv/production/soarSeeing.py @@ -0,0 +1,74 @@ +# This file is part of rubintv_production. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +from __future__ import annotations + +__all__ = [ + "SoarUploader", +] + +import logging +import tempfile +from time import sleep + +import matplotlib.pyplot as plt +from astropy.time import Time + +from lsst.rubintv.production.uploaders import MultiUploader +from lsst.rubintv.production.utils import getRubinTvInstrumentName, raiseIf +from lsst.summit.extras.soarSeeing import SoarSeeingMonitor +from lsst.summit.utils.utils import getCurrentDayObs_int + + +class SoarUploader: + def __init__(self, doRaise: bool) -> None: + self.soar = SoarSeeingMonitor() + self.instrument = "LSSTComCam" # change to LSSTCam when we switch instruments + self.doRaise = doRaise + self.figure = plt.figure(figsize=(18, 10)) + self.uploader = MultiUploader() + self._lastUpdated = Time.now() + self.log = logging.getLogger(__name__) + + def newData(self) -> bool: + return self._lastUpdated != self.soar.getMostRecentTimestamp() + + def run(self) -> None: + while True: + dayObs = getCurrentDayObs_int() + try: + if not self.newData(): + sleep(15) + continue + + self._lastUpdated = self.soar.getMostRecentTimestamp() + self.figure = self.soar.plotSeeingForDayObs(dayObs, addMostRecentBox=False, fig=self.figure) + with tempfile.NamedTemporaryFile(suffix=".png") as f: + self.log.info(f"Uploading SOAR seeing to night report for {dayObs}") + self.figure.savefig(f.name) + self.uploader.uploadNightReportData( + getRubinTvInstrumentName(self.instrument), dayObs, f.name, "SoarSeeingMonitor" + ) + self.log.info("Done") + self.figure.clear() + + except Exception as e: + logging.error(f"Error: {e}") + raiseIf(self.doRaise, e, self.log) diff --git a/scripts/summit/misc/runSoarSeeingScraper.py b/scripts/summit/misc/runSoarSeeingScraper.py new file mode 100644 index 000000000..22166188b --- /dev/null +++ b/scripts/summit/misc/runSoarSeeingScraper.py @@ -0,0 +1,26 @@ +# This file is part of rubintv_production. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from lsst.summit.extras.soarSeeing import SoarDatabaseBuiler + +scraper = SoarDatabaseBuiler() + +scraper.run() diff --git a/scripts/summit/misc/runSoarSeeingUploader.py b/scripts/summit/misc/runSoarSeeingUploader.py new file mode 100644 index 000000000..62088c6e9 --- /dev/null +++ b/scripts/summit/misc/runSoarSeeingUploader.py @@ -0,0 +1,27 @@ +# This file is part of rubintv_production. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from lsst.rubintv.production.soarSeeing import SoarUploader +from lsst.rubintv.production.utils import getDoRaise + +uploader = SoarUploader(doRaise=getDoRaise()) + +uploader.run() From e68a351b982e6c0b71e55cdc52fcf05f2183f823 Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Mon, 2 Dec 2024 18:13:34 -0800 Subject: [PATCH 2/6] Simplify night report upload code --- python/lsst/rubintv/production/uploaders.py | 33 +++++++++++---------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/python/lsst/rubintv/production/uploaders.py b/python/lsst/rubintv/production/uploaders.py index 7ef84c28e..0f707a319 100644 --- a/python/lsst/rubintv/production/uploaders.py +++ b/python/lsst/rubintv/production/uploaders.py @@ -569,7 +569,10 @@ def uploadNightReportData( instrument: str, dayObs: int, filename: str, + uploadAs: str, plotGroup: str = None, + *, + isMetadataFile: bool = False, ) -> str: """Upload night report type plot or json file to a night report channel. @@ -582,11 +585,17 @@ def uploadNightReportData( The dayObs. filename : `str` The full path and filename of the file to upload. + uploadAs : `str` + The name of the desination file. Only include the last part, i.e. + how it should appear when displayed. plotGroup : `str`, optional The group to upload the plot to. The 'default' group is used if this is not specified. However, people are encouraged to supply groups for their plots, so the 'default' value is not put in the function signature to indicate this. + isMetadataFile : `bool`, optional + If the file is the md.json file, set this to True. Will ignore the + plotGroup and uploadAs parameters. Raises ------ @@ -610,29 +619,23 @@ def uploadNightReportData( if plotGroup is None: plotGroup = "default" - basename = os.path.basename(filename) dayObsStr = dayObsIntToString(dayObs) + baseName = f"{instrument}/{dayObsStr}/night_report" - if basename == "md.json": # it's not a plot, so special case this one case - uploadAs = ( - f"{instrument}/{dayObsStr}/night_report/{instrument}_night_report_{dayObsStr}_{basename}" - ) + if isMetadataFile: + destName = f"{baseName}/{instrument}_night_report_{dayObsStr}_md.json" else: - # the plot filenames have the channel name saved into them in the - # form path/channelName-plotName.png, so remove the channel name - # and dash - plotName = basename.replace(instrument + "_night_reports" + "-", "") - plotFilename = f"{instrument}_night_report_{dayObsStr}_{plotGroup}_{plotName}" - uploadAs = f"{instrument}/{dayObsStr}/night_report/{plotGroup}/{plotFilename}" + plotFilename = f"{instrument}_night_report_{dayObsStr}_{plotGroup}_{uploadAs}" + destName = f"{baseName}/{plotGroup}/{plotFilename}" try: - self.upload(destinationFilename=uploadAs, sourceFilename=filename) - self._log.info(f"Uploaded {filename} to {uploadAs}") + self.upload(destinationFilename=destName, sourceFilename=filename) + self._log.info(f"Uploaded {filename} to {destName}") except Exception as ex: - self._log.exception(f"Failed to upload {filename} as {uploadAs} for {instrument} night report") + self._log.exception(f"Failed to upload {filename} as {destName} for {instrument} night report") raise ex - return uploadAs + return destName @override def upload(self, destinationFilename: str, sourceFilename: str) -> str: From 871b000a666dc9804d5f1e8225345db398251a66 Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Mon, 2 Dec 2024 18:21:24 -0800 Subject: [PATCH 3/6] Make use of new night report upload code --- .../production/plotting/nightReportPlotBase.py | 12 ++++++++++-- python/lsst/rubintv/production/rubinTv.py | 8 +++++++- python/lsst/rubintv/production/uploaders.py | 2 +- tests/test_s3_uploader.py | 1 + 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/python/lsst/rubintv/production/plotting/nightReportPlotBase.py b/python/lsst/rubintv/production/plotting/nightReportPlotBase.py index 410310345..ff73615b0 100644 --- a/python/lsst/rubintv/production/plotting/nightReportPlotBase.py +++ b/python/lsst/rubintv/production/plotting/nightReportPlotBase.py @@ -179,7 +179,11 @@ def createAndUpload(self, nightReport, metadata, ccdVisitTable): plt.close() self.s3Uploader.uploadNightReportData( - instrument="auxtel", dayObs=self.dayObs, filename=saveFile, plotGroup=self.plotGroup + instrument="auxtel", + dayObs=self.dayObs, + filename=saveFile, + plotGroup=self.plotGroup, + uploadAs=self.plotName, ) # if things start failing later you don't want old plots sticking # around and getting re-uploaded as if they were new @@ -247,7 +251,11 @@ def createAndUpload(self, tableData): plt.close() self.s3Uploader.uploadNightReportData( - instrument="startracker", dayObs=self.dayObs, filename=saveFile, plotGroup=self.plotGroup + instrument="startracker", + dayObs=self.dayObs, + filename=saveFile, + plotGroup=self.plotGroup, + uploadAs=self.plotName, ) # if things start failing later you don't want old plots sticking # around and getting re-uploaded as if they were new diff --git a/python/lsst/rubintv/production/rubinTv.py b/python/lsst/rubintv/production/rubinTv.py index 798f3d56b..b7a244e68 100644 --- a/python/lsst/rubintv/production/rubinTv.py +++ b/python/lsst/rubintv/production/rubinTv.py @@ -1272,7 +1272,11 @@ def callback(self, expRecord, doCheckDay=True): airMassPlotFile = os.path.join(self.locationConfig.nightReportPath, "airmass.png") self.report.plotPerObjectAirMass(saveFig=airMassPlotFile) self.s3Uploader.uploadNightReportData( - instrument="auxtel", dayObs=self.dayObs, filename=airMassPlotFile, plotGroup="Coverage" + instrument="auxtel", + dayObs=self.dayObs, + filename=airMassPlotFile, + plotGroup="Coverage", + uploadAs="airmass.png", ) # the alt/az coverage polar plot @@ -1283,6 +1287,7 @@ def callback(self, expRecord, doCheckDay=True): dayObs=self.dayObs, filename=altAzCoveragePlotFile, plotGroup="Coverage", + uploadsAs="alt-az.png", ) # Add text items here @@ -1302,6 +1307,7 @@ def callback(self, expRecord, doCheckDay=True): instrument="auxtel", dayObs=self.dayObs, filename=jsonFilename, + isMetadataFile=True, ) self.log.info(f"Finished updating plots and table for {dataId}") diff --git a/python/lsst/rubintv/production/uploaders.py b/python/lsst/rubintv/production/uploaders.py index 0f707a319..6aed86736 100644 --- a/python/lsst/rubintv/production/uploaders.py +++ b/python/lsst/rubintv/production/uploaders.py @@ -587,7 +587,7 @@ def uploadNightReportData( The full path and filename of the file to upload. uploadAs : `str` The name of the desination file. Only include the last part, i.e. - how it should appear when displayed. + how it should appear when displayed, including the extension. plotGroup : `str`, optional The group to upload the plot to. The 'default' group is used if this is not specified. However, people are encouraged to supply diff --git a/tests/test_s3_uploader.py b/tests/test_s3_uploader.py index 8d813ccdd..acf108ac1 100644 --- a/tests/test_s3_uploader.py +++ b/tests/test_s3_uploader.py @@ -86,6 +86,7 @@ def test_uploadNightReportData(self) -> None: dayObs=observationDay, filename=filename, plotGroup=plotGroup, + uploadAs="test_file_0001.txt", ) self.is_correct_check_uploaded_file(uploadedFile, fileContent) From b4821162f0415737de4d2e7e17f944f3ae3bf442 Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Mon, 2 Dec 2024 18:25:03 -0800 Subject: [PATCH 4/6] Remove unused code and update interface --- python/lsst/rubintv/production/uploaders.py | 93 ++++++--------------- 1 file changed, 27 insertions(+), 66 deletions(-) diff --git a/python/lsst/rubintv/production/uploaders.py b/python/lsst/rubintv/production/uploaders.py index 6aed86736..2e39a73b8 100644 --- a/python/lsst/rubintv/production/uploaders.py +++ b/python/lsst/rubintv/production/uploaders.py @@ -32,7 +32,7 @@ from boto3.session import Session as S3_session from botocore.config import Config from botocore.exceptions import BotoCoreError, ClientError -from typing_extensions import Optional, override +from typing_extensions import override from lsst.summit.utils.utils import dayObsIntToString, getSite @@ -194,32 +194,50 @@ def __init__(self): @abstractmethod def uploadNightReportData( - self, channel: str, dayObs: int, filename: str, plotGroup: Optional[str] + self, + instrument: str, + dayObs: int, + filename: str, + uploadAs: str, + plotGroup: str = None, + *, + isMetadataFile: bool = False, ) -> str: - """Upload night report type plot or json file to a night - report channel. + """Upload night report type plot or json file + to a night report channel. Parameters ---------- - channel : `str` - The RubinTV channel to upload to. - observation_day : `int` + instrument : `str` + The instrument. + dayObs : `int` The dayObs. filename : `str` The full path and filename of the file to upload. + uploadAs : `str` + The name of the desination file. Only include the last part, i.e. + how it should appear when displayed, including the extension. plotGroup : `str`, optional The group to upload the plot to. The 'default' group is used if this is not specified. However, people are encouraged to supply groups for their plots, so the 'default' value is not put in the function signature to indicate this. + isMetadataFile : `bool`, optional + If the file is the md.json file, set this to True. Will ignore the + plotGroup and uploadAs parameters. Raises ------ ValueError Raised if the specified channel is not in the list of existing - channels as specified in CHANNELS. + channels as specified in CHANNELS UploadError - Raised if uploading the file to the Bucket was not possible. + Raised if uploading the file to the Bucket was not possible + + Returns + ------- + uploadAs: `str`` + Path and filename for the destination file in the bucket """ raise NotImplementedError() @@ -804,63 +822,6 @@ def uploadPerSeqNumPlot( return blob - def uploadNightReportData( - self, - channel, - dayObs, - filename, - plotGroup="", - ): - """Upload night report type plot or json file to a night report channel - - Parameters - ---------- - channel : `str` - The RubinTV channel to upload to. - dayObsInt : `int` - The dayObs. - filename : `str` - The full path and filename of the file to upload. - plotGroup : `str`, optional - The group to upload the plot to. The 'default' group is used if - this is not specified. However, people are encouraged to supply - groups for their plots, so the 'default' value is not put in the - function signature to indicate this. - - Raises - ------ - ValueError - Raised if the specified channel is not in the list of existing - channels as specified in CHANNELS - """ - if channel not in CHANNELS: - raise ValueError(f"Error: {channel} not in {CHANNELS}") - - basename = os.path.basename(filename) # deals with png vs jpeg - - # the plot filenames have the channel name saved into them in the form - # path/channelName-plotName.png, so remove the channel name and dash - basename = basename.replace(channel + "-", "") - uploadAs = f"{channel}/{dayObs}/{plotGroup if plotGroup else 'default'}/{basename}" - - blob = self.bucket.blob(uploadAs) - blob.cache_control = "no-store" - - # retry strategy here is also gentle as these plots are routintely - # updated, so we'll get a new version soon enough. - timeout = 60 # default is 60s - deadline = 2.0 - modified_retry = DEFAULT_RETRY.with_deadline(deadline) # in seconds - modified_retry = modified_retry.with_delay(initial=0.5, multiplier=1.2, maximum=2) - try: - blob.upload_from_filename(filename, retry=modified_retry, timeout=timeout) - self.log.info(f"Uploaded {filename} to {uploadAs}") - except Exception as e: - self.log.warning(f"Failed to upload {uploadAs} to {channel} because {repr(e)}") - return None - - return blob - def googleUpload( self, channel, From e7a6ba3de290fd0548648e54f764144803e07edf Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Mon, 2 Dec 2024 19:19:50 -0800 Subject: [PATCH 5/6] Add suffix to plot name and use kwargs in new upload func --- python/lsst/rubintv/production/soarSeeing.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/lsst/rubintv/production/soarSeeing.py b/python/lsst/rubintv/production/soarSeeing.py index ed438329e..a11e47679 100644 --- a/python/lsst/rubintv/production/soarSeeing.py +++ b/python/lsst/rubintv/production/soarSeeing.py @@ -64,7 +64,11 @@ def run(self) -> None: self.log.info(f"Uploading SOAR seeing to night report for {dayObs}") self.figure.savefig(f.name) self.uploader.uploadNightReportData( - getRubinTvInstrumentName(self.instrument), dayObs, f.name, "SoarSeeingMonitor" + instrument=getRubinTvInstrumentName(self.instrument), + dayObs=dayObs, + filename=f.name, + uploadAs="SoarSeeingMonitor.png", + plotGroup="Seeing", ) self.log.info("Done") self.figure.clear() From c1befbb4dc42fe3ee7f0288b4ecd707d5cc22c91 Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Mon, 2 Dec 2024 20:37:55 -0800 Subject: [PATCH 6/6] Add updated text box to upload --- python/lsst/rubintv/production/soarSeeing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lsst/rubintv/production/soarSeeing.py b/python/lsst/rubintv/production/soarSeeing.py index a11e47679..dbb9cff8a 100644 --- a/python/lsst/rubintv/production/soarSeeing.py +++ b/python/lsst/rubintv/production/soarSeeing.py @@ -59,7 +59,7 @@ def run(self) -> None: continue self._lastUpdated = self.soar.getMostRecentTimestamp() - self.figure = self.soar.plotSeeingForDayObs(dayObs, addMostRecentBox=False, fig=self.figure) + self.figure = self.soar.plotSeeingForDayObs(dayObs, addMostRecentBox=True, fig=self.figure) with tempfile.NamedTemporaryFile(suffix=".png") as f: self.log.info(f"Uploading SOAR seeing to night report for {dayObs}") self.figure.savefig(f.name)