From fcd34d54dd651066381ccfa8bd5179ff7b4bca39 Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Tue, 2 Jan 2024 12:24:52 -0300 Subject: [PATCH 1/3] changes to step --- magstats_step/magstats_step/core/magstats.py | 2 +- magstats_step/magstats_step/core/objstats.py | 4 ++-- magstats_step/magstats_step/step.py | 20 ++++++++++---------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/magstats_step/magstats_step/core/magstats.py b/magstats_step/magstats_step/core/magstats.py index db960c305..e66bc1508 100644 --- a/magstats_step/magstats_step/core/magstats.py +++ b/magstats_step/magstats_step/core/magstats.py @@ -7,7 +7,7 @@ class MagnitudeStatistics(BaseStatistics): - _JOIN = ["aid", "sid", "fid"] + _JOIN = ["oid", "sid", "fid"] # Saturation threshold for each survey (only applies to corrected magnitudes) _THRESHOLD = {"ZTF": 13.2} diff --git a/magstats_step/magstats_step/core/objstats.py b/magstats_step/magstats_step/core/objstats.py index d021e7666..be96da0c8 100644 --- a/magstats_step/magstats_step/core/objstats.py +++ b/magstats_step/magstats_step/core/objstats.py @@ -7,7 +7,7 @@ class ObjectStatistics(BaseStatistics): - _JOIN = "aid" + _JOIN = "oid" def __init__(self, detections: List[dict]): super().__init__(detections) @@ -47,7 +47,7 @@ def average(series): # Needs wrapper to use the sigmas in the agg call return self._weighted_mean(series, sigmas.loc[series.index]) sigmas = self._arcsec2deg(self._detections[f"e_{label}"]) - grouped_sigmas = self._group(sigmas.set_axis(self._detections["aid"])) + grouped_sigmas = self._group(sigmas.set_axis(self._detections["oid"])) return pd.DataFrame( { f"mean{label}": self._grouped_detections()[label].agg(average), diff --git a/magstats_step/magstats_step/step.py b/magstats_step/magstats_step/step.py index 9ff9431be..752d495fb 100644 --- a/magstats_step/magstats_step/step.py +++ b/magstats_step/magstats_step/step.py @@ -35,12 +35,12 @@ def _execute(self, messages: dict): magstats = magstats_calculator.generate_statistics( self.excluded ).reset_index() - magstats = magstats.set_index("aid").replace({np.nan: None}) - for aid in stats: + magstats = magstats.set_index("oid").replace({np.nan: None}) + for oid in stats: try: - stats[aid]["magstats"] = magstats.loc[aid].to_dict("records") + stats[oid]["magstats"] = magstats.loc[oid].to_dict("records") except TypeError: - stats[aid]["magstats"] = [magstats.loc[aid].to_dict()] + stats[oid]["magstats"] = [magstats.loc[oid].to_dict()] return stats @@ -63,12 +63,12 @@ def _execute_ztf(self, messages: dict): magstats = magstats_calculator.generate_statistics( self.excluded ).reset_index() - magstats = magstats.set_index("aid").replace({np.nan: None}) - for aid in stats: + magstats = magstats.set_index("oid").replace({np.nan: None}) + for oid in stats: try: - stats[aid]["magstats"] = magstats.loc[aid].to_dict("records") + stats[oid]["magstats"] = magstats.loc[oid].to_dict("records") except TypeError: - stats[aid]["magstats"] = [magstats.loc[aid].to_dict()] + stats[oid]["magstats"] = [magstats.loc[oid].to_dict()] return stats @@ -80,11 +80,11 @@ def execute(self, messages: dict): # it seems that we'll have to produce different commands in this def produce_scribe(self, result: dict): - for aid, stats in result.items(): + for oid, stats in result.items(): command = { "collection": "object", "type": "update", - "criteria": {"_id": aid}, + "criteria": {"oid": oid}, "data": stats | { "loc": { From c53b9c9ab37dc6d5a1be0c853fff5fcce8e2560b Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Tue, 2 Jan 2024 16:12:32 -0300 Subject: [PATCH 2/3] now passing tests --- magstats_step/tests/integration/conftest.py | 11 +- magstats_step/tests/integration/test_step.py | 2 +- .../tests/unittests/data/messages.py | 21 ++-- .../tests/unittests/test_magstats.py | 118 ++++++++++++++---- .../tests/unittests/test_objstats.py | 72 ++++++----- magstats_step/tests/unittests/test_step.py | 49 ++++---- 6 files changed, 187 insertions(+), 86 deletions(-) diff --git a/magstats_step/tests/integration/conftest.py b/magstats_step/tests/integration/conftest.py index 842bf118f..2ce004d8e 100644 --- a/magstats_step/tests/integration/conftest.py +++ b/magstats_step/tests/integration/conftest.py @@ -1,6 +1,7 @@ import pytest import uuid import os +import pathlib from confluent_kafka.admin import AdminClient, NewTopic from apf.producers import KafkaProducer @@ -56,7 +57,7 @@ def env_variables(): env_variables_dict = { "PRODUCER_SCHEMA_PATH": "", "CONSUMER_SCHEMA_PATH": "", - "METRIS_SCHEMA_PATH": "../schemas/magstats_step//metrics.json", + "METRIS_SCHEMA_PATH": "../schemas/magstats_step/metrics.json", "SCRIBE_SCHEMA_PATH": "../schemas/scribe_step/scribe.avsc", "CONSUMER_SERVER": "localhost:9092", "CONSUMER_TOPICS": "correction", @@ -95,8 +96,12 @@ def produce_messages(topic): { "PARAMS": {"bootstrap.servers": "localhost:9092"}, "TOPIC": topic, - "SCHEMA_PATH": os.path.join( - os.path.dirname(__file__), "../../schema.avsc" + "SCHEMA_PATH": str( + pathlib.Path( + pathlib.Path(__file__).parent.parent.parent.parent, + "schemas/correction_step", + "output.avsc", + ) ), } ) diff --git a/magstats_step/tests/integration/test_step.py b/magstats_step/tests/integration/test_step.py index 4941b9cf3..6e026aff7 100644 --- a/magstats_step/tests/integration/test_step.py +++ b/magstats_step/tests/integration/test_step.py @@ -9,7 +9,7 @@ def assert_message_schema(command): assert "oid" in command["criteria"] elif command["collection"] == "object": assert command["type"] == "update" - assert "_id" in command["criteria"] + assert "oid" in command["criteria"] else: assert False assert "data" in command diff --git a/magstats_step/tests/unittests/data/messages.py b/magstats_step/tests/unittests/data/messages.py index d08a2b588..042d0e9ae 100644 --- a/magstats_step/tests/unittests/data/messages.py +++ b/magstats_step/tests/unittests/data/messages.py @@ -1,9 +1,18 @@ import random +import pathlib from fastavro import schema from fastavro import utils -SCHEMA = schema.load_schema("schema.avsc") +SCHEMA_PATH = str( + pathlib.Path( + pathlib.Path(__file__).parent.parent.parent.parent.parent, + "schemas/correction_step", + "output.avsc", + ) +) + +SCHEMA = schema.load_schema(SCHEMA_PATH) random.seed(42) aids_pool = [f"AID22X{i}" for i in range(10)] @@ -12,20 +21,18 @@ data = list(utils.generate_many(SCHEMA, 10)) for d in data: aid = random.choice(aids_pool) - d["aid"] = aid - sid = "ZTF" if random.random() < 0.5 else "ATLAS" oid = random.choice(oids_pool) + d["oid"] = oid + sid = "ZTF" if random.random() < 0.5 else "ATLAS" for detection in d["detections"]: detection["aid"] = aid + detection["oid"] = oid detection["sid"] = sid detection["fid"] = "g" if random.random() < 0.5 else "r" detection["forced"] = False - if sid == "ZTF": - detection["oid"] = oid for non_detection in d["non_detections"]: non_detection["aid"] = aid + non_detection["oid"] = oid non_detection["sid"] = sid non_detection["fid"] = "g" if random.random() < 0.5 else "r" - if sid == "ZTF": - non_detection["oid"] = oid diff --git a/magstats_step/tests/unittests/test_magstats.py b/magstats_step/tests/unittests/test_magstats.py index 237d8f891..e7c380105 100644 --- a/magstats_step/tests/unittests/test_magstats.py +++ b/magstats_step/tests/unittests/test_magstats.py @@ -10,6 +10,7 @@ def test_calculate_uncorrected_stats_gives_statistics_for_magnitudes_per_aid_and detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mag": 2, @@ -18,6 +19,7 @@ def test_calculate_uncorrected_stats_gives_statistics_for_magnitudes_per_aid_and }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mag": 2, @@ -26,6 +28,7 @@ def test_calculate_uncorrected_stats_gives_statistics_for_magnitudes_per_aid_and }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mag": 5, @@ -34,6 +37,7 @@ def test_calculate_uncorrected_stats_gives_statistics_for_magnitudes_per_aid_and }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "mag": 1, @@ -42,6 +46,7 @@ def test_calculate_uncorrected_stats_gives_statistics_for_magnitudes_per_aid_and }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mag": 1, @@ -50,6 +55,7 @@ def test_calculate_uncorrected_stats_gives_statistics_for_magnitudes_per_aid_and }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mag": 2, @@ -67,13 +73,13 @@ def test_calculate_uncorrected_stats_gives_statistics_for_magnitudes_per_aid_and "magmax": [5, 1, 2], "magmin": [2, 1, 1], "magsigma": [np.sqrt(2), 0, 0.5], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -81,6 +87,7 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mag_corr": 2, @@ -90,6 +97,7 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mag_corr": 2, @@ -99,6 +107,7 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mag_corr": 5, @@ -108,6 +117,7 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mag_corr": 5, @@ -117,6 +127,7 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "mag_corr": 1, @@ -126,6 +137,7 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mag_corr": 1, @@ -135,6 +147,7 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mag_corr": 2, @@ -144,6 +157,7 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 2, "mag_corr": 2, @@ -162,13 +176,13 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per "magmax_corr": [5, 1, 2], "magmin_corr": [2, 1, 1], "magsigma_corr": [np.sqrt(2), 0, 0.5], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -176,6 +190,7 @@ def test_calculate_uncorrected_stats_over_time_gives_first_and_last_magnitude_pe detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 3, @@ -185,6 +200,7 @@ def test_calculate_uncorrected_stats_over_time_gives_first_and_last_magnitude_pe }, # last { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -194,6 +210,7 @@ def test_calculate_uncorrected_stats_over_time_gives_first_and_last_magnitude_pe }, # first { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 2, @@ -203,6 +220,7 @@ def test_calculate_uncorrected_stats_over_time_gives_first_and_last_magnitude_pe }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -212,6 +230,7 @@ def test_calculate_uncorrected_stats_over_time_gives_first_and_last_magnitude_pe }, # last and first { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 1, @@ -221,6 +240,7 @@ def test_calculate_uncorrected_stats_over_time_gives_first_and_last_magnitude_pe }, # first { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 2, @@ -236,13 +256,13 @@ def test_calculate_uncorrected_stats_over_time_gives_first_and_last_magnitude_pe { "magfirst": [2, 1, 1], "maglast": [1, 1, 2], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -250,6 +270,7 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 3, @@ -260,6 +281,7 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -270,6 +292,7 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 2, @@ -280,6 +303,7 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 4, @@ -290,6 +314,7 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -300,6 +325,7 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 1, @@ -310,6 +336,7 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 2, @@ -320,6 +347,7 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn }, { "aid": "AID2", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 0, @@ -336,13 +364,13 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn { "magfirst_corr": [2, 1, 1], "maglast_corr": [1, 1, 2], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -364,6 +392,7 @@ def test_calculate_firstmjd_gives_first_date_per_aid_and_fid(): detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 3, @@ -372,6 +401,7 @@ def test_calculate_firstmjd_gives_first_date_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 0, @@ -380,6 +410,7 @@ def test_calculate_firstmjd_gives_first_date_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 2, @@ -388,6 +419,7 @@ def test_calculate_firstmjd_gives_first_date_per_aid_and_fid(): }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "mjd": 0.5, @@ -396,6 +428,7 @@ def test_calculate_firstmjd_gives_first_date_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 1, @@ -404,6 +437,7 @@ def test_calculate_firstmjd_gives_first_date_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 2, @@ -417,13 +451,13 @@ def test_calculate_firstmjd_gives_first_date_per_aid_and_fid(): expected = pd.DataFrame( { "firstmjd": [0, 0.5, 1], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -431,6 +465,7 @@ def test_calculate_lastmjd_gives_last_date_per_aid_and_fid(): detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 3, @@ -439,6 +474,7 @@ def test_calculate_lastmjd_gives_last_date_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -447,6 +483,7 @@ def test_calculate_lastmjd_gives_last_date_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 2, @@ -455,6 +492,7 @@ def test_calculate_lastmjd_gives_last_date_per_aid_and_fid(): }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -463,6 +501,7 @@ def test_calculate_lastmjd_gives_last_date_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 1, @@ -471,6 +510,7 @@ def test_calculate_lastmjd_gives_last_date_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 2, @@ -484,13 +524,13 @@ def test_calculate_lastmjd_gives_last_date_per_aid_and_fid(): expected = pd.DataFrame( { "lastmjd": [3, 1, 2], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -498,6 +538,7 @@ def test_calculate_corrected_gives_whether_first_detection_per_aid_and_fid_is_co detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 3, @@ -507,6 +548,7 @@ def test_calculate_corrected_gives_whether_first_detection_per_aid_and_fid_is_co }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -516,6 +558,7 @@ def test_calculate_corrected_gives_whether_first_detection_per_aid_and_fid_is_co }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 2, @@ -525,6 +568,7 @@ def test_calculate_corrected_gives_whether_first_detection_per_aid_and_fid_is_co }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -534,6 +578,7 @@ def test_calculate_corrected_gives_whether_first_detection_per_aid_and_fid_is_co }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 1, @@ -543,6 +588,7 @@ def test_calculate_corrected_gives_whether_first_detection_per_aid_and_fid_is_co }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 2, @@ -557,13 +603,13 @@ def test_calculate_corrected_gives_whether_first_detection_per_aid_and_fid_is_co expected = pd.DataFrame( { "corrected": [False, True, True], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -571,6 +617,7 @@ def test_calculate_stellar_gives_whether_first_detection_per_aid_and_fid_is_stel detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 3, @@ -580,6 +627,7 @@ def test_calculate_stellar_gives_whether_first_detection_per_aid_and_fid_is_stel }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -589,6 +637,7 @@ def test_calculate_stellar_gives_whether_first_detection_per_aid_and_fid_is_stel }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 2, @@ -598,6 +647,7 @@ def test_calculate_stellar_gives_whether_first_detection_per_aid_and_fid_is_stel }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -607,6 +657,7 @@ def test_calculate_stellar_gives_whether_first_detection_per_aid_and_fid_is_stel }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 1, @@ -616,6 +667,7 @@ def test_calculate_stellar_gives_whether_first_detection_per_aid_and_fid_is_stel }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 2, @@ -630,13 +682,13 @@ def test_calculate_stellar_gives_whether_first_detection_per_aid_and_fid_is_stel expected = pd.DataFrame( { "stellar": [False, True, True], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -644,6 +696,7 @@ def test_calculate_ndet_gives_number_of_detections_per_aid_and_fid(): detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "candid": "a", @@ -651,6 +704,7 @@ def test_calculate_ndet_gives_number_of_detections_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "candid": "b", @@ -658,6 +712,7 @@ def test_calculate_ndet_gives_number_of_detections_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "candid": "c", @@ -665,6 +720,7 @@ def test_calculate_ndet_gives_number_of_detections_per_aid_and_fid(): }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "candid": "d", @@ -672,6 +728,7 @@ def test_calculate_ndet_gives_number_of_detections_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "candid": "e", @@ -679,6 +736,7 @@ def test_calculate_ndet_gives_number_of_detections_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "candid": "f", @@ -691,13 +749,13 @@ def test_calculate_ndet_gives_number_of_detections_per_aid_and_fid(): expected = pd.DataFrame( { "ndet": [3, 1, 2], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -705,6 +763,7 @@ def test_calculate_ndubious_gives_number_of_dubious_detections_per_aid_and_fid() detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "dubious": True, @@ -713,6 +772,7 @@ def test_calculate_ndubious_gives_number_of_dubious_detections_per_aid_and_fid() }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "dubious": True, @@ -721,6 +781,7 @@ def test_calculate_ndubious_gives_number_of_dubious_detections_per_aid_and_fid() }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "dubious": False, @@ -729,6 +790,7 @@ def test_calculate_ndubious_gives_number_of_dubious_detections_per_aid_and_fid() }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "dubious": False, @@ -737,6 +799,7 @@ def test_calculate_ndubious_gives_number_of_dubious_detections_per_aid_and_fid() }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "dubious": True, @@ -745,6 +808,7 @@ def test_calculate_ndubious_gives_number_of_dubious_detections_per_aid_and_fid() }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "dubious": False, @@ -758,13 +822,13 @@ def test_calculate_ndubious_gives_number_of_dubious_detections_per_aid_and_fid() expected = pd.DataFrame( { "ndubious": [2, 0, 1], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -772,6 +836,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "ZTF", "fid": 1, "corrected": True, @@ -781,6 +846,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "ZTF", "fid": 1, "corrected": True, @@ -790,6 +856,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "ZTF", "fid": 1, "corrected": True, @@ -799,6 +866,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "ZTF", "fid": 1, "corrected": True, @@ -808,6 +876,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, { "aid": "AID2", + "oid": "OID2", "sid": "ZTF", "fid": 2, "corrected": False, @@ -817,6 +886,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, { "aid": "AID2", + "oid": "OID2", "sid": "ZTF", "fid": 3, "corrected": False, @@ -826,6 +896,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, { "aid": "AID2", + "oid": "OID2", "sid": "ZTF", "fid": 3, "corrected": True, @@ -835,6 +906,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 10, "corrected": True, @@ -844,6 +916,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, # No threshold { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 10, "corrected": True, @@ -853,6 +926,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, # No threshold { "aid": "AID3", + "oid": "OID3", "sid": "SURVEY", "fid": 1, "corrected": False, @@ -867,13 +941,13 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): expected = pd.DataFrame( { "saturation_rate": [0.5, np.nan, 0, np.nan, np.nan], - "aid": ["AID1", "AID2", "AID2", "AID1", "AID3"], + "oid": ["OID1", "OID2", "OID2", "OID1", "OID3"], "sid": ["ZTF", "ZTF", "ZTF", "SURVEY", "SURVEY"], "fid": [1, 2, 3, 10, 1], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) diff --git a/magstats_step/tests/unittests/test_objstats.py b/magstats_step/tests/unittests/test_objstats.py index 172f8f977..3b08d54da 100644 --- a/magstats_step/tests/unittests/test_objstats.py +++ b/magstats_step/tests/unittests/test_objstats.py @@ -73,9 +73,9 @@ def test_calculate_weighted_mean_error_with_one_very_large_error_has_that_error_ def test_calculate_coordinates_with_ra_uses_weighted_mean_and_weighted_mean_error_per_aid(): detections = [ - {"aid": "AID1", "ra": 10, "e_ra": 2, "candid": "a", "forced": False}, - {"aid": "AID2", "ra": 20, "e_ra": 4, "candid": "c", "forced": False}, - {"aid": "AID1", "ra": 20, "e_ra": 4, "candid": "b", "forced": False}, + {"aid": "AID1", "oid": "OID1", "ra": 10, "e_ra": 2, "candid": "a", "forced": False}, + {"aid": "AID2", "oid": "OID2", "ra": 10, "ra": 20, "e_ra": 4, "candid": "c", "forced": False}, + {"aid": "AID1", "oid": "OID1", "ra": 10, "ra": 20, "e_ra": 4, "candid": "b", "forced": False}, ] calculator = ObjectStatistics(detections) @@ -130,9 +130,9 @@ def test_calculate_coordinates_with_ra_uses_weighted_mean_and_weighted_mean_erro def test_calculate_coordinates_with_dec_uses_weighted_mean_and_weighted_mean_error_per_aid(): detections = [ - {"aid": "AID1", "dec": 10, "e_dec": 2, "candid": "a", "forced": False}, - {"aid": "AID2", "dec": 20, "e_dec": 4, "candid": "c", "forced": False}, - {"aid": "AID1", "dec": 20, "e_dec": 4, "candid": "b", "forced": False}, + {"aid": "AID1", "oid": "AID1", "dec": 10, "e_dec": 2, "candid": "a", "forced": False}, + {"aid": "AID2", "oid": "AID2", "dec": 20, "e_dec": 4, "candid": "c", "forced": False}, + {"aid": "AID1", "oid": "AID1", "dec": 20, "e_dec": 4, "candid": "b", "forced": False}, ] calculator = ObjectStatistics(detections) @@ -187,10 +187,10 @@ def test_calculate_coordinates_with_dec_uses_weighted_mean_and_weighted_mean_err def test_calculate_unique_gives_list_of_unique_values_in_field_per_aid(): detections = [ - {"aid": "AID1", "candid": "a", "extra": "A", "forced": False}, - {"aid": "AID2", "candid": "c", "extra": "A", "forced": False}, - {"aid": "AID1", "candid": "b", "extra": "A", "forced": False}, - {"aid": "AID1", "candid": "d", "extra": "B", "forced": False}, + {"aid": "AID1", "oid": "OID1", "candid": "a", "extra": "A", "forced": False}, + {"aid": "AID2", "oid": "OID2", "candid": "c", "extra": "A", "forced": False}, + {"aid": "AID1", "oid": "OID1", "candid": "b", "extra": "A", "forced": False}, + {"aid": "AID1", "oid": "OID1", "candid": "d", "extra": "B", "forced": False}, ] calculator = ObjectStatistics(detections) result = calculator._calculate_unique("extra") @@ -200,7 +200,7 @@ def test_calculate_unique_gives_list_of_unique_values_in_field_per_aid(): result["extra"], pd.Series( [["A", "B"], ["A"]], - index=pd.Index(["AID1", "AID2"], name="aid"), + index=pd.Index(["OID1", "OID2"], name="oid"), name="extra", ), ) @@ -228,10 +228,10 @@ def test_calculate_dec_uses_calculate_coordinates(): def test_calculate_ndet_gives_number_of_detections_per_aid(): detections = [ - {"aid": "AID1", "candid": "a", "forced": False}, - {"aid": "AID2", "candid": "c", "forced": False}, - {"aid": "AID1", "candid": "b", "forced": False}, - {"aid": "AID1", "candid": "d", "forced": False}, + {"aid": "AID1", "oid": "OID1", "candid": "a", "forced": False}, + {"aid": "AID2", "oid": "OID2", "candid": "c", "forced": False}, + {"aid": "AID1", "oid": "OID1", "candid": "b", "forced": False}, + {"aid": "AID1", "oid": "OID1", "candid": "d", "forced": False}, ] calculator = ObjectStatistics(detections) result = calculator.calculate_ndet() @@ -240,17 +240,17 @@ def test_calculate_ndet_gives_number_of_detections_per_aid(): assert_series_equal( result["ndet"], pd.Series( - [3, 1], index=pd.Index(["AID1", "AID2"], name="aid"), name="ndet" + [3, 1], index=pd.Index(["OID1", "OID2"], name="oid"), name="ndet" ), ) def test_calculate_firstmjd_gives_the_first_mjd_per_aid(): detections = [ - {"aid": "AID1", "mjd": 1, "candid": "a", "forced": False}, - {"aid": "AID2", "mjd": 2, "candid": "c", "forced": False}, - {"aid": "AID1", "mjd": 3, "candid": "b", "forced": False}, - {"aid": "AID1", "mjd": 2, "candid": "d", "forced": False}, + {"aid": "AID1", "oid": "OID1", "mjd": 1, "candid": "a", "forced": False}, + {"aid": "AID2", "oid": "OID2", "mjd": 2, "candid": "c", "forced": False}, + {"aid": "AID1", "oid": "OID1", "mjd": 3, "candid": "b", "forced": False}, + {"aid": "AID1", "oid": "OID1", "mjd": 2, "candid": "d", "forced": False}, ] calculator = ObjectStatistics(detections) result = calculator.calculate_firstmjd() @@ -260,7 +260,7 @@ def test_calculate_firstmjd_gives_the_first_mjd_per_aid(): result["firstmjd"], pd.Series( [1, 2], - index=pd.Index(["AID1", "AID2"], name="aid"), + index=pd.Index(["OID1", "OID2"], name="oid"), name="firstmjd", ), ) @@ -268,10 +268,10 @@ def test_calculate_firstmjd_gives_the_first_mjd_per_aid(): def test_calculate_lastmjd_gives_the_last_mjd_per_aid(): detections = [ - {"aid": "AID1", "mjd": 1, "candid": "a", "forced": False}, - {"aid": "AID2", "mjd": 2, "candid": "c", "forced": False}, - {"aid": "AID1", "mjd": 3, "candid": "b", "forced": False}, - {"aid": "AID1", "mjd": 2, "candid": "d", "forced": False}, + {"aid": "AID1", "oid": "OID1", "mjd": 1, "candid": "a", "forced": False}, + {"aid": "AID2", "oid": "OID2", "mjd": 2, "candid": "c", "forced": False}, + {"aid": "AID1", "oid": "OID1", "mjd": 3, "candid": "b", "forced": False}, + {"aid": "AID1", "oid": "OID1", "mjd": 2, "candid": "d", "forced": False}, ] calculator = ObjectStatistics(detections) result = calculator.calculate_lastmjd() @@ -281,7 +281,7 @@ def test_calculate_lastmjd_gives_the_last_mjd_per_aid(): result["lastmjd"], pd.Series( [3, 2], - index=pd.Index(["AID1", "AID2"], name="aid"), + index=pd.Index(["OID1", "OID2"], name="oid"), name="lastmjd", ), ) @@ -311,6 +311,7 @@ def test_calculate_corrected_gives_whether_first_detection_in_surveys_with_corre detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "MOCK_SURVEY", "mjd": 1, "corrected": False, @@ -319,6 +320,7 @@ def test_calculate_corrected_gives_whether_first_detection_in_surveys_with_corre }, # Should ignore { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "mjd": 2, "corrected": True, @@ -327,6 +329,7 @@ def test_calculate_corrected_gives_whether_first_detection_in_surveys_with_corre }, # True for AID1 { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "mjd": 3, "corrected": False, @@ -335,6 +338,7 @@ def test_calculate_corrected_gives_whether_first_detection_in_surveys_with_corre }, { "aid": "AID2", + "oid": "OID2", "sid": "MOCK_SURVEY", "mjd": 1, "corrected": True, @@ -343,6 +347,7 @@ def test_calculate_corrected_gives_whether_first_detection_in_surveys_with_corre }, # Should ignore { "aid": "AID3", + "oid": "OID3", "sid": "SURVEY", "mjd": 2, "corrected": False, @@ -351,6 +356,7 @@ def test_calculate_corrected_gives_whether_first_detection_in_surveys_with_corre }, # False for AID3 { "aid": "AID3", + "oid": "OID3", "sid": "SURVEY", "mjd": 3, "corrected": True, @@ -367,7 +373,7 @@ def test_calculate_corrected_gives_whether_first_detection_in_surveys_with_corre result["corrected"], pd.Series( [True, False], - index=pd.Index(["AID1", "AID3"], name="aid"), + index=pd.Index(["OID1", "OID3"], name="oid"), name="corrected", ), ) @@ -377,6 +383,7 @@ def test_calculate_stellar_gives_whether_first_detection_in_surveys_with_stellar detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "MOCK_SURVEY", "mjd": 1, "stellar": False, @@ -385,6 +392,7 @@ def test_calculate_stellar_gives_whether_first_detection_in_surveys_with_stellar }, # Should ignore { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "mjd": 2, "stellar": True, @@ -393,6 +401,7 @@ def test_calculate_stellar_gives_whether_first_detection_in_surveys_with_stellar }, # True for AID1 { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "mjd": 3, "stellar": False, @@ -401,6 +410,7 @@ def test_calculate_stellar_gives_whether_first_detection_in_surveys_with_stellar }, { "aid": "AID2", + "oid": "OID2", "sid": "MOCK_SURVEY", "mjd": 1, "stellar": True, @@ -409,6 +419,7 @@ def test_calculate_stellar_gives_whether_first_detection_in_surveys_with_stellar }, # Should ignore { "aid": "AID3", + "oid": "OID3", "sid": "SURVEY", "mjd": 2, "stellar": False, @@ -417,6 +428,7 @@ def test_calculate_stellar_gives_whether_first_detection_in_surveys_with_stellar }, # False for AID3 { "aid": "AID3", + "oid": "OID3", "sid": "SURVEY", "mjd": 3, "stellar": True, @@ -433,7 +445,7 @@ def test_calculate_stellar_gives_whether_first_detection_in_surveys_with_stellar result["stellar"], pd.Series( [True, False], - index=pd.Index(["AID1", "AID3"], name="aid"), + index=pd.Index(["OID1", "OID3"], name="oid"), name="stellar", ), ) @@ -458,6 +470,7 @@ def test_object_statistics_deltajd(): detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "mjd": 1, "stellar": False, @@ -466,6 +479,7 @@ def test_object_statistics_deltajd(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "mjd": 5, "stellar": True, @@ -477,5 +491,5 @@ def test_object_statistics_deltajd(): result = calculator.calculate_deltajd() assert_series_equal( result["deltajd"], - pd.Series([4], index=pd.Index(["AID1"], name="aid"), name="deltajd"), + pd.Series([4], index=pd.Index(["OID1"], name="oid"), name="deltajd"), ) diff --git a/magstats_step/tests/unittests/test_step.py b/magstats_step/tests/unittests/test_step.py index b7749acf9..621ca3a41 100644 --- a/magstats_step/tests/unittests/test_step.py +++ b/magstats_step/tests/unittests/test_step.py @@ -12,18 +12,19 @@ def test_execute_multistream(env_variables): formatted_data = step.pre_execute(data) result = step.execute(formatted_data) result = result["multistream"] + print(result.keys()) for d in data: - assert d["aid"] in result - assert "meanra" in result[d["aid"]] - assert "meandec" in result[d["aid"]] - assert "magstats" in result[d["aid"]] - assert "oid" in result[d["aid"]] - assert "tid" in result[d["aid"]] - assert "firstmjd" in result[d["aid"]] - assert "lastmjd" in result[d["aid"]] - assert "ndet" in result[d["aid"]] - assert "sigmara" in result[d["aid"]] - assert "sigmadec" in result[d["aid"]] + assert d["oid"] in result + assert "meanra" in result[d["oid"]] + assert "meandec" in result[d["oid"]] + assert "magstats" in result[d["oid"]] + assert "oid" in result[d["oid"]] + assert "tid" in result[d["oid"]] + assert "firstmjd" in result[d["oid"]] + assert "lastmjd" in result[d["oid"]] + assert "ndet" in result[d["oid"]] + assert "sigmara" in result[d["oid"]] + assert "sigmadec" in result[d["oid"]] def test_scribe_message_multistream(env_variables): @@ -36,7 +37,7 @@ def test_scribe_message_multistream(env_variables): step.post_execute(result) result = result["multistream"] for d in data: - to_write = result[d["aid"]] + to_write = result[d["oid"]] to_write.update( { "loc": { @@ -51,7 +52,7 @@ def test_scribe_message_multistream(env_variables): command = { "collection": "object", "type": "update", - "criteria": {"_id": d["aid"]}, + "criteria": {"oid": d["oid"]}, "data": to_write, "options": {"upsert": True}, } @@ -68,14 +69,14 @@ def test_execute_ztf(env_variables): for d in data: if not any([det["sid"] == "ZTF" for det in d["detections"]]): continue - assert d["aid"] in result - assert "meanra" in result[d["aid"]] - assert "meandec" in result[d["aid"]] - assert "magstats" in result[d["aid"]] - assert "oid" in result[d["aid"]] - assert "tid" in result[d["aid"]] - assert "firstmjd" in result[d["aid"]] - assert "lastmjd" in result[d["aid"]] - assert "ndet" in result[d["aid"]] - assert "sigmara" in result[d["aid"]] - assert "sigmadec" in result[d["aid"]] + assert d["oid"] in result + assert "meanra" in result[d["oid"]] + assert "meandec" in result[d["oid"]] + assert "magstats" in result[d["oid"]] + assert "oid" in result[d["oid"]] + assert "tid" in result[d["oid"]] + assert "firstmjd" in result[d["oid"]] + assert "lastmjd" in result[d["oid"]] + assert "ndet" in result[d["oid"]] + assert "sigmara" in result[d["oid"]] + assert "sigmadec" in result[d["oid"]] From 7d210ea43aeaca3ecd32621f501c8a567e974d7f Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Tue, 2 Jan 2024 16:23:15 -0300 Subject: [PATCH 3/3] reverted changes to scribe parser criteria --- magstats_step/magstats_step/step.py | 4 ++-- magstats_step/tests/integration/test_step.py | 4 ++-- magstats_step/tests/unittests/test_step.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/magstats_step/magstats_step/step.py b/magstats_step/magstats_step/step.py index 752d495fb..b2bd4e25c 100644 --- a/magstats_step/magstats_step/step.py +++ b/magstats_step/magstats_step/step.py @@ -84,7 +84,7 @@ def produce_scribe(self, result: dict): command = { "collection": "object", "type": "update", - "criteria": {"oid": oid}, + "criteria": {"_id": oid}, "data": stats | { "loc": { @@ -107,7 +107,7 @@ def produce_scribe_ztf(self, result: dict): { "collection": "magstats", "type": "upsert", - "criteria": {"oid": oid}, + "criteria": {"_id": oid}, "data": stats, } for oid in oids diff --git a/magstats_step/tests/integration/test_step.py b/magstats_step/tests/integration/test_step.py index 6e026aff7..77fb6d80d 100644 --- a/magstats_step/tests/integration/test_step.py +++ b/magstats_step/tests/integration/test_step.py @@ -6,10 +6,10 @@ def assert_message_schema(command): if command["collection"] == "magstats": assert command["type"] == "upsert" - assert "oid" in command["criteria"] + assert "_id" in command["criteria"] elif command["collection"] == "object": assert command["type"] == "update" - assert "oid" in command["criteria"] + assert "_id" in command["criteria"] else: assert False assert "data" in command diff --git a/magstats_step/tests/unittests/test_step.py b/magstats_step/tests/unittests/test_step.py index 621ca3a41..92e6891c5 100644 --- a/magstats_step/tests/unittests/test_step.py +++ b/magstats_step/tests/unittests/test_step.py @@ -52,7 +52,7 @@ def test_scribe_message_multistream(env_variables): command = { "collection": "object", "type": "update", - "criteria": {"oid": d["oid"]}, + "criteria": {"_id": d["oid"]}, "data": to_write, "options": {"upsert": True}, }