From a1cb2bc27eb2cf71cfc57899065f6bb5aae7cd99 Mon Sep 17 00:00:00 2001 From: MatteoGiomi Date: Wed, 30 Jan 2019 11:33:38 +0100 Subject: [PATCH] can pass global SCIENCEPROGRAM ID to ProgramList. No need to hardcode --- marshaltools/ProgramList.py | 47 +++++++++++++++++++++++++++++++------ marshaltools/gci_utils.py | 23 +++++++++++------- 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/marshaltools/ProgramList.py b/marshaltools/ProgramList.py index 396dd98..da207ba 100644 --- a/marshaltools/ProgramList.py +++ b/marshaltools/ProgramList.py @@ -111,6 +111,11 @@ def __init__(self, program, load_sources=True, load_candidates=False, sfd_dir=No self.logger.info("Initialized ProgramList for program %s (ID %d)"%(self.program, self.programidx)) self._dustmap = None + # this is the general, 'program wide' ID used to query for candidates. + # stars with a None, and it will be read from marshaltools.gci_utils.SCIENCEPROGRAM_IDS + # use set_querycandidates_programid method to change its value + self.science_program_id = None + # now load all the saved sources if load_sources: self.get_saved_sources() @@ -119,6 +124,30 @@ def __init__(self, program, load_sources=True, load_candidates=False, sfd_dir=No self.lightcurves = None + def set_querycandidates_programid(self, science_program_id): + """ + apparently the program ID you use to ingest/save candidates + is different from the one you use to query the candidate page. + In particular, the first one seems to be 'user specific', while + the second one is 'program specific'. This is, at least, what I + have understood for now. + + Some of these 'program specific' IDs are listed in marshaltools.gci_utils.SCIENCEPROGRAM_IDS. + If your channel is not there, you can use this function to set + it on the fly. + + Parameters: + ----------- + + science_program_id: `int` + 'program-specific' ID, used, e.g. when querying for candidates. + """ + + self.science_program_id = science_program_id + self.logger.debug("setting SCIENCE program ID for program %s to %d"% + (self.program, self.science_program_id)) + + def ingest_avro(self, avro_id, be_anal=True, max_attempts=3): """ ingest alert(s) into the marshall. @@ -146,7 +175,8 @@ def ingest_avro(self, avro_id, be_anal=True, max_attempts=3): avro_ids = avro_id, program_name = self.program, program_id = self.programidx, - be_anal = be_anal, + be_anal = be_anal, + query_program_id = self.science_program_id, max_attempts = max_attempts, auth=(self.user, self.passwd), logger=self.logger @@ -181,11 +211,14 @@ def save_sources(self, candidate, programidx=None, save_by='name', max_attempts= # if you don't pass the programID go read it from the static list of program-names & ids. if programidx is None: - programidx = SCIENCEPROGRAM_IDS.get(self.program, -666) - if programidx == -666: - raise KeyError("program %s is not listed in `marshaltools.gci_utils.SCIENCEPROGRAM_IDS`. Go and add it yourself!") - self.logger.info("reading programid from `marshaltools.gci_utils.SCIENCEPROGRAM_IDS`") - self.logger.info("programid for saving candidates for program %s: %d"%(self.program, programidx)) + if self.science_program_id is None: + programidx = SCIENCEPROGRAM_IDS.get(self.program, -666) + if programidx == -666: + raise KeyError("program %s is not listed in `marshaltools.gci_utils.SCIENCEPROGRAM_IDS`. Go and add it yourself!") + self.logger.debug("reading programid from `marshaltools.gci_utils.SCIENCEPROGRAM_IDS`") + else: + programidx = self.science_program_id + self.logger.info("programid for saving candidates for program %s: %d"%(self.program, programidx)) # parse save_by to cgi acceptable key if save_by == 'name': @@ -678,13 +711,13 @@ def query_candidate_page(self, showsaved, start_date=None, end_date=None): start_date = "2018-03-01 00:00:00" if end_date is None: end_date = Time.now().datetime.strftime("%Y-%m-%d %H:%M:%S") - return query_scanning_page( start_date, end_date, program_name=self.program, showsaved=showsaved, auth=(self.user, self.passwd), + program_id = self.programidx, logger=self.logger) diff --git a/marshaltools/gci_utils.py b/marshaltools/gci_utils.py index ef07a70..6bb9d4b 100644 --- a/marshaltools/gci_utils.py +++ b/marshaltools/gci_utils.py @@ -48,7 +48,8 @@ 'Redshift Completeness Factor' : 24, 'Weizmann Test Filter' : 45, 'ZTFBH Offnucear' : 47, - 'Nuclear Transients' : 10 + 'Nuclear Transients' : 10, + #'AmpelRapid' : 67 } INGEST_PROGRAM_IDS = { # TODO add all of them @@ -130,20 +131,25 @@ def growthcgi(scriptname, to_json=True, logger=None, max_attemps=2, **request_kw return rinfo -def query_scanning_page(start_date, end_date, program_name, showsaved="selected", auth=None, logger=None): +def query_scanning_page(start_date, end_date, program_name, showsaved="selected", auth=None, logger=None, program_id=None): """ return the sources in the scanning page of the given program ingested in the marshall between start_date and end_date. """ + # TODO: the ID used to query the scanning page seems to be different from + # the one you get from lits programs. + # get the logger logger = logger if not logger is None else logging.getLogger(__name__) # get scienceprogram number - scienceprogram = SCIENCEPROGRAM_IDS.get(program_name) - if scienceprogram is None: - raise KeyError("cannot find scienceprogram number corresponding to program %s. We have: %s"% - (program_name, repr(SCIENCEPROGRAM_IDS))) + scienceprogram = program_id + if program_id is None: + scienceprogram = SCIENCEPROGRAM_IDS.get(program_name) + if scienceprogram is None: + raise KeyError("cannot find scienceprogram number corresponding to program %s. We have: %s"% + (program_name, repr(SCIENCEPROGRAM_IDS))) # format dates to astropy.Time tstart = Time(start_date) if type(start_date) is str else start_date @@ -170,7 +176,7 @@ def query_scanning_page(start_date, end_date, program_name, showsaved="selected" return srcs -def ingest_candidates(avro_ids, program_name, program_id, be_anal, max_attempts=3, auth=None, logger=None): +def ingest_candidates(avro_ids, program_name, program_id, query_program_id, be_anal, max_attempts=3, auth=None, logger=None): """ ingest one or more candidate(s) by avro id into the marhsal. If needed we can be anal about it and go and veryfy the ingestion. @@ -239,7 +245,8 @@ def ingest_candidates(avro_ids, program_name, program_id, be_anal, max_attempts= start_date=start_ingestion.iso, end_date=end_ingestion.iso, program_name=program_name, - showsaved="selected", + showsaved="selected", + program_id=query_program_id, auth=auth, logger=logger)