BTS model v1 + extra ML features (#243)

* update the BTS model
* extra ML features

config.defaults.yaml

@@ -1118,7 +1118,7 @@ kowalski:
   ml:
     ZTF:
       # instruments need to have a list of allowed features (as tuple), and a list of models
-      allowed_features: ('drb', 'diffmaglim', 'ra', 'dec', 'magpsf', 'sigmapsf', 'chipsf', 'fwhm', 'sky', 'chinr', 'sharpnr', 'sgscore1', 'distpsnr1', 'sgscore2', 'distpsnr2', 'sgscore3', 'distpsnr3', 'ndethist', 'ncovhist', 'scorr', 'nmtchps', 'clrcoeff', 'clrcounc', 'neargaia', 'neargaiabright', 'classtar', 'peakmag', 'age')
+      allowed_features: ('drb', 'diffmaglim', 'ra', 'dec', 'magpsf', 'sigmapsf', 'chipsf', 'fwhm', 'sky', 'chinr', 'sharpnr', 'sgscore1', 'distpsnr1', 'sgscore2', 'distpsnr2', 'sgscore3', 'distpsnr3', 'ndethist', 'ncovhist', 'scorr', 'nmtchps', 'clrcoeff', 'clrcounc', 'neargaia', 'neargaiabright', 'classtar', 'peakmag_so_far', 'maxmag_so_far', 'days_since_peak', 'days_to_peak', 'nnondet', 'age')
       models:
       # models need: a version (string, e.g. "v1"), a triplet (bool), and feature_names (bool, or list of feature names as tuple)
       # if feature_names is True, all features from allowed_features are used
@@ -1152,13 +1152,13 @@ kowalski:
           feature_names: ('drb', 'diffmaglim', 'ra', 'dec', 'magpsf', 'sigmapsf', 'chipsf', 'fwhm', 'sky', 'chinr', 'sharpnr', 'sgscore1', 'distpsnr1', 'sgscore2', 'distpsnr2', 'sgscore3', 'distpsnr3', 'ndethist', 'ncovhist', 'scorr', 'nmtchps', 'clrcoeff', 'clrcounc', 'neargaia', 'neargaiabright')
           version: "d1_dnn_20201130"
           url: "https://github.com/dmitryduev/acai/raw/master/models/acai_b.d1_dnn_20201130.h5"
-        # bts:
-        #   triplet: True
-        #   feature_names: ('sgscore1','distpsnr1','sgscore2','distpsnr2','fwhm','magpsf','sigmapsf','ra','dec','diffmaglim','ndethist','nmtchps','age','peakmag')
-        #   version: "v03"
-        #   format: "pb"
-        #   order: ["triplet", "features"]
-        #   url: "https://raw.githubusercontent.com/nabeelre/BNB-models/main/v03.tar.gz"
+        bts:
+          triplet: True
+          feature_names: ('sgscore1', 'distpsnr1', 'sgscore2', 'distpsnr2', 'fwhm', 'magpsf', 'sigmapsf', 'chipsf', 'ra', 'dec', 'diffmaglim', 'ndethist', 'nmtchps', 'age', 'days_since_peak', 'days_to_peak', 'peakmag_so_far', 'drb', 'ncovhist', 'nnondet', 'chinr', 'sharpnr', 'scorr', 'sky', 'maxmag_so_far')
+          version: "v1"
+          format: "pb"
+          order: ["triplet", "features"]
+          url: "https://raw.githubusercontent.com/nabeelre/BTSbot/main/production_models/v1.tar.gz"
 
 
   skyportal:

kowalski/tests/test_alert_broker_ztf.py

@@ -3,6 +3,7 @@
 from kowalski.alert_brokers.alert_broker_ztf import ZTFAlertWorker
 from kowalski.config import load_config
 from kowalski.log import log
+from copy import deepcopy
 
 """ load config and secrets """
 config = load_config(config_files=["config.yaml"])["kowalski"]
@@ -109,10 +110,11 @@ def test_make_thumbnails(self):
 
     def test_alert_filter__ml(self):
         """Test executing ML models on the alert"""
-        alert, _ = self.worker.alert_mongify(self.alert)
-        scores = self.worker.alert_filter__ml(alert)
+        alert, prv_candidates = self.worker.alert_mongify(self.alert)
+        all_prv_candidates = deepcopy(prv_candidates) + [deepcopy(alert["candidate"])]
+        scores = self.worker.alert_filter__ml(alert, all_prv_candidates)
         assert len(scores) > 0
-        log(scores)
+        # print(scores)
 
     def test_alert_filter__xmatch(self):
         """Test cross matching with external catalog"""

kowalski/utils.py

@@ -1176,19 +1176,48 @@ def __init__(self, alert, alert_history, models, label=None, **kwargs):
 
         self.alert = deepcopy(alert)
 
-        # add a peakmag field to the alert (min of all magpsf)
-        self.alert["candidate"]["peakmag"] = min(
-            [30]
-            + [
-                a.get("magpsf", 30)
-                for a in alert_history
-                if a.get("magpsf", None) is not None
-            ]
+        # ADD EXTRA FEATURES
+        peakmag_jd = alert["candidate"]["jd"]
+        peakmag = 30
+        maxmag = 0
+        # find the mjd of the peak magnitude
+        for a in alert_history:
+            if a.get("magpsf", None) is not None:
+                if a["magpsf"] < peakmag:
+                    peakmag = a["magpsf"]
+                    peakmag_jd = a["jd"]
+                if a["magpsf"] > maxmag:
+                    maxmag = a["magpsf"]
+
+        first_alert_jd = min(
+            alert["candidate"].get("jdstarthist", None),
+            min(
+                [alert["candidate"]["jd"]]
+                + [a["jd"] for a in alert_history if a["magpsf"] is not None]
+            ),
         )
-        # add an age field to the alert (alert["candidate"].jd - alert["candidate"].jdstarthist)
-        self.alert["candidate"]["age"] = self.alert["candidate"]["jd"] - self.alert[
-            "candidate"
-        ].get("jdstarthist", self.alert["candidate"]["jd"])
+
+        # add a peakmag_so_far field to the alert (min of all magpsf)
+        self.alert["candidate"]["peakmag_so_far"] = peakmag
+
+        # add a maxmag_so_far field to the alert (max of all magpsf)
+        self.alert["candidate"]["maxmag_so_far"] = maxmag
+
+        # add a days_since_peak field to the alert (jd - peakmag_jd)
+        self.alert["candidate"]["days_since_peak"] = (
+            self.alert["candidate"]["jd"] - peakmag_jd
+        )
+
+        # add a days_to_peak field to the alert (peakmag_jd - first_alert_jd)
+        self.alert["candidate"]["days_to_peak"] = peakmag_jd - first_alert_jd
+
+        # add an age field to the alert: (jd - first_alert_jd)
+        self.alert["candidate"]["age"] = self.alert["candidate"]["jd"] - first_alert_jd
+
+        # number of non-detections: ncovhist - ndethist
+        self.alert["candidate"]["nnondet"] = alert["candidate"].get(
+            "ncovhist", 0
+        ) - alert["candidate"].get("ndethist", 0)
 
         triplet_normalize = kwargs.get("triplet_normalize", True)
         to_tpu = kwargs.get("to_tpu", False)
@@ -1204,6 +1233,8 @@ def __init__(self, alert, alert_history, models, label=None, **kwargs):
             # "dmdt": dmdt
         }
 
+        del peakmag_jd, peakmag, maxmag, first_alert_jd
+
     def make_triplet(self, normalize: bool = True, to_tpu: bool = False):
         """
         Feed in alert packet