diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml index 078e868..2c6a250 100644 --- a/.github/workflows/integration_test.yml +++ b/.github/workflows/integration_test.yml @@ -16,25 +16,6 @@ jobs: matrix: python-version: [3.6, 3.7, 3.8] - # Service containers to run with `container-job` - services: - # Label used to access the service container - postgres: - # Docker Hub image - image: postgres - # Provide the password for postgres - env: - POSTGRES_PASSWORD: postgres - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - # Maps tcp port 5432 on service container to the host - - 5432:5432 - steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -44,7 +25,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pytest numpy Cython + pip install pytest numpy Cython pytest-docker psycopg2 if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Test with pytest run: | diff --git a/docker-compose.yml b/docker-compose.yml index 464b978..3070c04 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,26 +1,27 @@ version: "3" services: - features: - build: - context: . - dockerfile: Dockerfile - restart: "always" - env_file: - - db_settings.env - - es_settings.env + feature: + build: . environment: - - CONSUMER_TOPICS= - - CONSUMER_SERVER= - - CONSUMER_GROUP_ID= - - PRODUCER_TOPIC= - - PRODUCER_SERVER= - - ES_PREFIX= - - ES_NETWORK_HOST= - - ES_NETWORK_PORT= - - DB_HOST= - - DB_USER= - - DB_PASSWORD= - - DB_PORT= - - DB_NAME= - - METRICS_HOST= - - METRICS_TOPIC= + - DB_ENGINE=postgres + - DB_HOST=${DB_HOST} + - DB_USER=${DB_USER} + - DB_PASSWORD=${DB_PASSWORD} + - DB_PORT=5432 + - DB_NAME=${DB_NAME} + - CONSUMER_TOPICS=preprocess + - CONSUMER_SERVER=${CONSUMER_SERVER} + - CONSUMER_GROUP_ID=features_consumer_batch + - PRODUCER_TOPIC=features_batch + - PRODUCER_SERVER=${PRODUCER_SERVER} + - STEP_VERSION=features_1.0.1_dev + - FEATURE_VERSION=features_1.0.1_dev + - METRICS_HOST=${METRICS_HOST} + - METRICS_TOPIC=${METRICS_TOPIC} + - CONSUME_TIMEOUT=60 + - CONSUME_MESSAGES=10 + volumes: + - ./:/app + stdin_open: true + tty: true + command: /bin/bash diff --git a/features/step.py b/features/step.py index fca0bb4..4e6ecc5 100644 --- a/features/step.py +++ b/features/step.py @@ -13,6 +13,7 @@ from lc_classifier.features.custom import CustomStreamHierarchicalExtractor from pandas.io.json import json_normalize +from sqlalchemy.sql.expression import bindparam warnings.filterwarnings("ignore") logging.getLogger("GP").setLevel(logging.WARNING) @@ -56,7 +57,6 @@ def __init__( if not step_args.get("test_mode", False): self.insert_step_metadata() - def insert_step_metadata(self): self.db.query(Step).get_or_create( filter_by={"step_id": self.config["STEP_METADATA"]["STEP_ID"]}, @@ -66,68 +66,6 @@ def insert_step_metadata(self): date=datetime.datetime.now(), ) - def preprocess_detections(self, detections): - """ - Preprocess detections. As of version 1.0.0 it only converts detections to dataframe. - """ - detections = self.create_detections_dataframe(detections) - return detections - - def create_detections_dataframe(self, detections): - """Format detections to `pandas.DataFrame`. - This method take detections in dict form and use `json_normalize` to transform it to a DataFrame. After that - rename some columns and object_id as index. - **Example:** - - Parameters - ---------- - detections : dict - dict with detections as they come from the preprocess step. - """ - detections = json_normalize(detections) - detections.rename( - columns={ - "alert.sgscore1": "sgscore1", - "alert.isdiffpos": "isdiffpos", - }, - inplace=True, - ) - detections.set_index("oid", inplace=True) - return detections - - def preprocess_non_detections(self, non_detections): - """ - Convert non detections to dataframe. - - Parameters - ---------- - non_detections : dict - non_detections as they come from preprocess step - """ - return json_normalize(non_detections) - - def preprocess_xmatches(self, xmatches): - """ - As of version 1.0.0 it does no preprocess operations on xmatches. - - Parameters - ---------- - xmatches : dict - xmatches as they come from preprocess step - """ - return xmatches - - def preprocess_metadata(self, metadata): - """ - As of version 1.0.0 it does no preprocess operations on alert metadata. - - Parameters - ---------- - metadata : dict - metadata as they come from preprocess step - """ - return metadata - def compute_features(self, detections, non_detections, metadata, xmatches): """Compute Hierarchical-Features in detections and non detections to `dict`. @@ -139,10 +77,10 @@ def compute_features(self, detections, non_detections, metadata, xmatches): Detections of an object non_detections : pandas.DataFrame Non detections of an object - metadata : dict + metadata : pandas.DataFrame Metadata from the alert with other catalogs info - obj : dict - Object data + xmatches : pandas.DataFrame + Xmatches data from xmatch step """ features = self.features_computer.compute_features( detections, @@ -154,48 +92,6 @@ def compute_features(self, detections, non_detections, metadata, xmatches): features = features.astype(float) return features - def insert_db(self, oid, result, preprocess_id): - """Insert the `dict` result in database. - Consider: - - object: Refer with oid - - features: In result `dict` - - version: Set in config of the step - - **Example:** - - Parameters - ---------- - oid : string - Object identifier of all detections - result : dict - Result of features compute - fid : pd.DataFrame - - """ - feature_version, created = self.db.query(FeatureVersion).get_or_create( - filter_by={ - "version": self.config["STEP_METADATA"]["FEATURE_VERSION"], - "step_id_feature": self.config["STEP_METADATA"]["STEP_ID"], - "step_id_preprocess": preprocess_id, - } - ) - for key in result: - fid = self.get_fid(key) - if fid < 0: - continue - feature, created = self.db.query(Feature).get_or_create( - filter_by={ - "oid": oid, - "name": key, - "fid": fid, - "version": feature_version.version, - }, - value=result[key], - ) - if not created: - self.db.query().update(feature, {"value": feature.value}) - self.db.session.commit() - def get_fid(self, feature): """ Gets the band number (fid) of a feature. @@ -214,6 +110,11 @@ def get_fid(self, feature): feature : str name of the feature """ + if not isinstance(feature, str): + self.logger.error( + f"Feature {feature} is not a valid feature. Should be str instance with fid after underscore (_)" + ) + return -99 fid0 = [ "W1", "W1-W2", @@ -238,6 +139,7 @@ def get_fid(self, feature): "g-r_max_corr", "g-r_mean", "g-r_mean_corr", + "PPE", ] if feature in fid0: return 0 @@ -248,43 +150,210 @@ def get_fid(self, feature): return int(fid) return -99 - def convert_nan(self, result): - """Changes nan values to None + def get_on_db(self, result): + oids = result.index.values + query = ( + self.db.query(Feature.oid) + .filter(Feature.oid.in_(oids)) + .filter(Feature.version == self.config["STEP_METADATA"]["STEP_VERSION"]) + .distinct() + ) + return pd.read_sql(query.statement, self.db.engine).oid.values + + def insert_feature_version(self, preprocess_id): + self.feature_version, created = self.db.query(FeatureVersion).get_or_create( + filter_by={ + "version": self.config["STEP_METADATA"]["FEATURE_VERSION"], + "step_id_feature": self.config["STEP_METADATA"]["STEP_ID"], + "step_id_preprocess": preprocess_id, + } + ) + + def update_db(self, to_update, out_columns, apply_get_fid): + if len(to_update) == 0: + return + self.logger.info(f"Updating {len(to_update)} features") + to_update.replace({np.nan: None}, inplace=True) + to_update = to_update.stack(dropna=False) + to_update = to_update.to_frame() + to_update.reset_index(inplace=True) + to_update.columns = out_columns + to_update["fid"] = to_update["name"].apply(apply_get_fid) + to_update["version"] = self.feature_version.version + to_update["name"] = to_update["name"].apply(lambda x: self.check_feature_name(x)) + to_update.rename( + columns={ + "oid": "_oid", + "fid": "_fid", + "version": "_version", + "name": "_name", + "value": "_value", + }, + inplace=True, + ) + dict_to_update = to_update.to_dict("records") + stmt = ( + Feature.__table__.update() + .where(Feature.oid == bindparam("_oid")) + .where(Feature.name == bindparam("_name")) + .where(Feature.fid == bindparam("_fid")) + .where(Feature.version == bindparam("_version")) + .values(value=bindparam("_value")) + ) + self.db.engine.execute(stmt, dict_to_update) + return dict_to_update + + def insert_db(self, to_insert, out_columns, apply_get_fid): + if len(to_insert) == 0: + return + self.logger.info(f"Inserting {len(to_insert)} new features") + to_insert.replace({np.nan: None}, inplace=True) + to_insert = to_insert.stack(dropna=False) + to_insert = to_insert.to_frame() + to_insert.reset_index(inplace=True) + to_insert.columns = out_columns + to_insert["fid"] = to_insert.name.apply(apply_get_fid) + to_insert["version"] = self.feature_version.version + to_insert["name"] = to_insert.name.apply(lambda x: self.check_feature_name(x)) + dict_to_insert = to_insert.to_dict("records") + self.db.query().bulk_insert(dict_to_insert, Feature) + return dict_to_insert + + def check_feature_name(self, name): + fid = name.rsplit("_", 1)[-1] + if fid.isdigit(): + return name.rsplit("_", 1)[0] + else: + return name + + def add_to_db(self, result): + """Insert the `dict` result in database. + Consider: + - object: Refer with oid + - features: In result `dict` + - version: Set in config of the step + + **Example:** Parameters ---------- result : dict - Dict that will have nans removed + Result of features compute """ - cleaned_results = {} - for key in result: - if type(result[key]) is dict: - cleaned_results[key] = self.convert_nan(result[key]) - else: - if np.isnan(result[key]): - cleaned_results[key] = None - else: - cleaned_results[key] = result[key] - return cleaned_results - - def execute(self, message): - oid = message["oid"] - detections = self.preprocess_detections(message["detections"]) - non_detections = self.preprocess_non_detections(message["non_detections"]) - xmatches = self.preprocess_xmatches(message["xmatches"]) - metadata = self.preprocess_metadata(message["metadata"]) - if len(detections) < 6: - self.logger.debug(f"{oid} Object has less than 6 detections") - return - self.logger.debug(f"{oid} Object has enough detections. Calculating Features") + out_columns = ["oid", "name", "value"] + on_db = self.get_on_db(result) + already_on_db = result.index.isin(on_db) + to_insert = result.loc[~already_on_db] + to_update = result.loc[already_on_db] + apply_get_fid = lambda x: self.get_fid(x) + + if len(to_update) > 0: + self.update_db(to_update, out_columns, apply_get_fid) + if len(to_insert) > 0: + self.insert_db(to_insert, out_columns, apply_get_fid) + + def produce(self, features, alert_data): + alert_data.set_index("oid", inplace=True) + alert_data.drop_duplicates(inplace=True, keep="last") + for oid, features_oid in features.iterrows(): + features_oid.replace({np.nan: None}, inplace=True) + candid = alert_data.loc[oid].candid + features_dict = features_oid.to_dict() + out_message = {"features": features_dict, "oid": oid, "candid": candid} + self.producer.produce(out_message, key=oid) + + def get_metadata_from_message(self, message): + return { + "oid": message["oid"], + "candid": message["candid"], + "sgscore1": message["metadata"]["ps1"]["sgscore1"], + } + + def get_xmatches_from_message(self, message): + if "xmatches" in message and message["xmatches"] is not None: + allwise = message["xmatches"].get("allwise") + xmatch_values = { + "W1mag": allwise["W1mag"], + "W2mag": allwise["W2mag"], + "W3mag": allwise["W3mag"], + } + else: + xmatch_values = {"W1mag": np.nan, "W2mag": np.nan, "W3mag": np.nan} + return {"oid": message["oid"], "candid": message["candid"], **xmatch_values} + + def delete_duplicate_detections(self, detections): + self.logger.debug(f"Before Dropping: {len(detections)} Detections") + detections.drop_duplicates(["oid", "candid"], inplace=True) + self.logger.debug(f"After Dropping: {len(detections)} Detections") + + def delete_duplicate_non_detections(self, non_detections): + self.logger.debug(f"Before Dropping: {len(non_detections)} Non Detections") + non_detections["round_mjd"] = non_detections.mjd.round(6) + non_detections.drop_duplicates(["oid", "round_mjd", "fid"], inplace=True) + self.logger.debug(f"After Dropping: {len(non_detections)} Non Detections") + + def delete_duplicates(self, detections, non_detections): + self.delete_duplicate_detections(detections) + self.delete_duplicate_non_detections(non_detections) + + def get_data_from_messages(self, messages): + """ + Gets detections, non_detections, metadata and xmatches from consumed messages + and converts them to pd.DataFrame + """ + detections = [] + non_detections = [] + metadata = [] + xmatches = [] + + for message in messages: + detections.extend(message.get("detections", [])) + non_detections.extend(message.get("non_detections", [])) + metadata.append(self.get_metadata_from_message(message)) + xmatches.append(self.get_xmatches_from_message(message)) + + return ( + pd.DataFrame(detections), + pd.DataFrame(non_detections), + pd.DataFrame(metadata), + pd.DataFrame(xmatches), + ) + + def execute(self, messages): + self.logger.info(f"Processing {len(messages)} messages.") + + preprocess_id = messages[0]["preprocess_step_id"] + self.insert_feature_version(preprocess_id) + + self.logger.info("Getting batch alert data") + alert_data = pd.DataFrame( + [ + {"oid": message.get("oid"), "candid": message.get("candid", np.nan)} + for message in messages + ] + ) + unique_oid = len(alert_data.oid.unique()) + self.logger.info(f"Found {unique_oid} Objects.") + + self.logger.info("Getting detections and non_detections") + + detections, non_detections, metadata, xmatches = self.get_data_from_messages( + messages + ) + + if unique_oid < len(messages): + self.delete_duplicates(detections, non_detections) + + if len(detections): + detections.set_index("oid", inplace=True) + if len(non_detections): + non_detections.set_index("oid", inplace=True) + + self.logger.info(f"Calculating features") features = self.compute_features(detections, non_detections, metadata, xmatches) - if len(features) <= 0: - self.logger.debug(f"No features for {oid}") - return - if type(features) is pd.Series: - features = pd.DataFrame([features]) - result = self.convert_nan(features.loc[oid].to_dict()) - self.insert_db(oid, result, message["preprocess_step_id"]) + + self.logger.info(f"Features calculated: {features.shape}") + if len(features) > 0: + self.add_to_db(features) if self.producer: - out_message = {"features": result, "candid": message["candid"], "oid": oid} - self.producer.produce(out_message, key=oid) + self.produce(features, alert_data) diff --git a/schema.py b/schema.py new file mode 100644 index 0000000..8a3a412 --- /dev/null +++ b/schema.py @@ -0,0 +1,226 @@ +SCHEMA = { + "doc": "Features", + "name": "features_document", + "type": "record", + "fields": [ + {"name": "oid", "type": "string"}, + {"name": "candid", "type": "long"}, + { + "name": "features", + "type": { + "name": "features_record", + "type": "record", + "fields": [ + {"name": "Amplitude_1", "type": ["float", "null"]}, + {"name": "Amplitude_2", "type": ["float", "null"]}, + {"name": "AndersonDarling_1", "type": ["float", "null"]}, + {"name": "AndersonDarling_2", "type": ["float", "null"]}, + {"name": "Autocor_length_1", "type": ["double", "null"]}, + {"name": "Autocor_length_2", "type": ["double", "null"]}, + {"name": "Beyond1Std_1", "type": ["float", "null"]}, + {"name": "Beyond1Std_2", "type": ["float", "null"]}, + {"name": "Con_1", "type": ["double", "null"]}, + {"name": "Con_2", "type": ["double", "null"]}, + {"name": "Eta_e_1", "type": ["float", "null"]}, + {"name": "Eta_e_2", "type": ["float", "null"]}, + {"name": "ExcessVar_1", "type": ["double", "null"]}, + {"name": "ExcessVar_2", "type": ["double", "null"]}, + {"name": "GP_DRW_sigma_1", "type": ["double", "null"]}, + {"name": "GP_DRW_sigma_2", "type": ["double", "null"]}, + {"name": "GP_DRW_tau_1", "type": ["float", "null"]}, + {"name": "GP_DRW_tau_2", "type": ["float", "null"]}, + {"name": "Gskew_1", "type": ["float", "null"]}, + {"name": "Gskew_2", "type": ["float", "null"]}, + {"name": "Harmonics_mag_1_1", "type": ["float", "null"]}, + {"name": "Harmonics_mag_1_2", "type": ["float", "null"]}, + {"name": "Harmonics_mag_2_1", "type": ["float", "null"]}, + {"name": "Harmonics_mag_2_2", "type": ["float", "null"]}, + {"name": "Harmonics_mag_3_1", "type": ["float", "null"]}, + {"name": "Harmonics_mag_3_2", "type": ["float", "null"]}, + {"name": "Harmonics_mag_4_1", "type": ["float", "null"]}, + {"name": "Harmonics_mag_4_2", "type": ["float", "null"]}, + {"name": "Harmonics_mag_5_1", "type": ["float", "null"]}, + {"name": "Harmonics_mag_5_2", "type": ["float", "null"]}, + {"name": "Harmonics_mag_6_1", "type": ["float", "null"]}, + {"name": "Harmonics_mag_6_2", "type": ["float", "null"]}, + {"name": "Harmonics_mag_7_1", "type": ["float", "null"]}, + {"name": "Harmonics_mag_7_2", "type": ["float", "null"]}, + {"name": "Harmonics_mse_1", "type": ["double", "null"]}, + {"name": "Harmonics_mse_2", "type": ["double", "null"]}, + {"name": "Harmonics_phase_2_1", "type": ["float", "null"]}, + {"name": "Harmonics_phase_2_2", "type": ["float", "null"]}, + {"name": "Harmonics_phase_3_1", "type": ["float", "null"]}, + {"name": "Harmonics_phase_3_2", "type": ["float", "null"]}, + {"name": "Harmonics_phase_4_1", "type": ["float", "null"]}, + {"name": "Harmonics_phase_4_2", "type": ["float", "null"]}, + {"name": "Harmonics_phase_5_1", "type": ["float", "null"]}, + {"name": "Harmonics_phase_5_2", "type": ["float", "null"]}, + {"name": "Harmonics_phase_6_1", "type": ["float", "null"]}, + {"name": "Harmonics_phase_6_2", "type": ["float", "null"]}, + {"name": "Harmonics_phase_7_1", "type": ["float", "null"]}, + {"name": "Harmonics_phase_7_2", "type": ["float", "null"]}, + {"name": "IAR_phi_1", "type": ["double", "null"]}, + {"name": "IAR_phi_2", "type": ["float", "null"]}, + {"name": "LinearTrend_1", "type": ["float", "null"]}, + {"name": "LinearTrend_2", "type": ["double", "null"]}, + {"name": "MHPS_PN_flag_1", "type": ["double", "null"]}, + {"name": "MHPS_PN_flag_2", "type": ["double", "null"]}, + {"name": "MHPS_high_1", "type": ["float", "null"]}, + {"name": "MHPS_high_2", "type": ["double", "null"]}, + {"name": "MHPS_low_1", "type": ["float", "null"]}, + {"name": "MHPS_low_2", "type": ["float", "null"]}, + {"name": "MHPS_non_zero_1", "type": ["double", "null"]}, + {"name": "MHPS_non_zero_2", "type": ["double", "null"]}, + {"name": "MHPS_ratio_1", "type": ["float", "null"]}, + {"name": "MHPS_ratio_2", "type": ["float", "null"]}, + {"name": "MaxSlope_1", "type": ["float", "null"]}, + {"name": "MaxSlope_2", "type": ["float", "null"]}, + {"name": "Mean_1", "type": ["float", "null"]}, + {"name": "Mean_2", "type": ["float", "null"]}, + {"name": "Meanvariance_1", "type": ["float", "null"]}, + {"name": "Meanvariance_2", "type": ["float", "null"]}, + {"name": "MedianAbsDev_1", "type": ["float", "null"]}, + {"name": "MedianAbsDev_2", "type": ["float", "null"]}, + {"name": "MedianBRP_1", "type": ["float", "null"]}, + {"name": "MedianBRP_2", "type": ["float", "null"]}, + {"name": "Multiband_period", "type": ["float", "null"]}, + {"name": "PairSlopeTrend_1", "type": ["float", "null"]}, + {"name": "PairSlopeTrend_2", "type": ["float", "null"]}, + {"name": "PercentAmplitude_1", "type": ["float", "null"]}, + {"name": "PercentAmplitude_2", "type": ["float", "null"]}, + {"name": "Period_band_1", "type": ["float", "null"]}, + {"name": "Period_band_2", "type": ["float", "null"]}, + {"name": "delta_period_1", "type": ["float", "null"]}, + {"name": "delta_period_2", "type": ["float", "null"]}, + {"name": "Period_fit", "type": ["float", "null"]}, + {"name": "Power_rate_1/2", "type": ["float", "null"]}, + {"name": "Power_rate_1/3", "type": ["float", "null"]}, + {"name": "Power_rate_1/4", "type": ["float", "null"]}, + {"name": "Power_rate_2", "type": ["float", "null"]}, + {"name": "Power_rate_3", "type": ["float", "null"]}, + {"name": "Power_rate_4", "type": ["float", "null"]}, + {"name": "Psi_CS_1", "type": ["float", "null"]}, + {"name": "Psi_CS_2", "type": ["float", "null"]}, + {"name": "Psi_eta_1", "type": ["float", "null"]}, + {"name": "Psi_eta_2", "type": ["float", "null"]}, + {"name": "Pvar_1", "type": ["float", "null"]}, + {"name": "Pvar_2", "type": ["float", "null"]}, + {"name": "Q31_1", "type": ["float", "null"]}, + {"name": "Q31_2", "type": ["float", "null"]}, + {"name": "Rcs_1", "type": ["float", "null"]}, + {"name": "Rcs_2", "type": ["float", "null"]}, + {"name": "SF_ML_amplitude_1", "type": ["float", "null"]}, + {"name": "SF_ML_amplitude_2", "type": ["float", "null"]}, + {"name": "SF_ML_gamma_1", "type": ["float", "null"]}, + {"name": "SF_ML_gamma_2", "type": ["float", "null"]}, + {"name": "SPM_A_1", "type": ["float", "null"]}, + {"name": "SPM_A_2", "type": ["float", "null"]}, + {"name": "SPM_beta_1", "type": ["float", "null"]}, + {"name": "SPM_beta_2", "type": ["float", "null"]}, + {"name": "SPM_chi_1", "type": ["float", "null"]}, + {"name": "SPM_chi_2", "type": ["float", "null"]}, + {"name": "SPM_gamma_1", "type": ["float", "null"]}, + {"name": "SPM_gamma_2", "type": ["float", "null"]}, + {"name": "SPM_t0_1", "type": ["float", "null"]}, + {"name": "SPM_t0_2", "type": ["float", "null"]}, + {"name": "SPM_tau_fall_1", "type": ["float", "null"]}, + {"name": "SPM_tau_fall_2", "type": ["float", "null"]}, + {"name": "SPM_tau_rise_1", "type": ["float", "null"]}, + {"name": "SPM_tau_rise_2", "type": ["float", "null"]}, + {"name": "Skew_1", "type": ["float", "null"]}, + {"name": "Skew_2", "type": ["float", "null"]}, + {"name": "SmallKurtosis_1", "type": ["float", "null"]}, + {"name": "SmallKurtosis_2", "type": ["float", "null"]}, + {"name": "Std_1", "type": ["float", "null"]}, + {"name": "Std_2", "type": ["float", "null"]}, + {"name": "StetsonK_1", "type": ["float", "null"]}, + {"name": "StetsonK_2", "type": ["float", "null"]}, + {"name": "W1-W2", "type": ["double", "null"]}, + {"name": "W2-W3", "type": ["double", "null"]}, + {"name": "delta_mag_fid_1", "type": ["float", "null"]}, + {"name": "delta_mag_fid_2", "type": ["float", "null"]}, + {"name": "delta_mjd_fid_1", "type": ["float", "null"]}, + {"name": "delta_mjd_fid_2", "type": ["float", "null"]}, + {"name": "dmag_first_det_fid_1", "type": ["double", "null"]}, + {"name": "dmag_first_det_fid_2", "type": ["double", "null"]}, + {"name": "dmag_non_det_fid_1", "type": ["double", "null"]}, + {"name": "dmag_non_det_fid_2", "type": ["double", "null"]}, + {"name": "first_mag_1", "type": ["float", "null"]}, + {"name": "first_mag_2", "type": ["float", "null"]}, + {"name": "g-W2", "type": ["double", "null"]}, + {"name": "g-W3", "type": ["double", "null"]}, + {"name": "g-r_max", "type": ["float", "null"]}, + {"name": "g-r_max_corr", "type": ["float", "null"]}, + {"name": "g-r_mean", "type": ["float", "null"]}, + {"name": "g-r_mean_corr", "type": ["float", "null"]}, + {"name": "gal_b", "type": ["float", "null"]}, + {"name": "gal_l", "type": ["float", "null"]}, + {"name": "iqr_1", "type": ["float", "null"]}, + {"name": "iqr_2", "type": ["float", "null"]}, + { + "name": "last_diffmaglim_before_fid_1", + "type": ["double", "null"], + }, + { + "name": "last_diffmaglim_before_fid_2", + "type": ["double", "null"], + }, + {"name": "last_mjd_before_fid_1", "type": ["double", "null"]}, + {"name": "last_mjd_before_fid_2", "type": ["double", "null"]}, + { + "name": "max_diffmaglim_after_fid_1", + "type": ["double", "null"], + }, + { + "name": "max_diffmaglim_after_fid_2", + "type": ["double", "null"], + }, + { + "name": "max_diffmaglim_before_fid_1", + "type": ["double", "null"], + }, + { + "name": "max_diffmaglim_before_fid_2", + "type": ["double", "null"], + }, + {"name": "mean_mag_1", "type": ["float","null"]}, + {"name": "mean_mag_2", "type": ["float","null"]}, + { + "name": "median_diffmaglim_after_fid_1", + "type": ["double", "null"], + }, + { + "name": "median_diffmaglim_after_fid_2", + "type": ["double", "null"], + }, + { + "name": "median_diffmaglim_before_fid_1", + "type": ["double", "null"], + }, + { + "name": "median_diffmaglim_before_fid_2", + "type": ["double", "null"], + }, + {"name": "min_mag_1", "type": ["float", "null"]}, + {"name": "min_mag_2", "type": ["float", "null"]}, + {"name": "n_det_1", "type": ["double", "null"]}, + {"name": "n_det_2", "type": ["double", "null"]}, + {"name": "n_neg_1", "type": ["double", "null"]}, + {"name": "n_neg_2", "type": ["double", "null"]}, + {"name": "n_non_det_after_fid_1", "type": ["double", "null"]}, + {"name": "n_non_det_after_fid_2", "type": ["double", "null"]}, + {"name": "n_non_det_before_fid_1", "type": ["double", "null"]}, + {"name": "n_non_det_before_fid_2", "type": ["double", "null"]}, + {"name": "n_pos_1", "type": ["double", "null"]}, + {"name": "n_pos_2", "type": ["double", "null"]}, + {"name": "positive_fraction_1", "type": ["double", "null"]}, + {"name": "positive_fraction_2", "type": ["double", "null"]}, + {"name": "r-W2", "type": ["double", "null"]}, + {"name": "r-W3", "type": ["double", "null"]}, + {"name": "rb", "type": ["float", "null"]}, + {"name": "sgscore1", "type": ["float", "null"]} + ], + }, + }, + ], +} diff --git a/settings.py b/settings.py index 1871de6..4016668 100644 --- a/settings.py +++ b/settings.py @@ -2,6 +2,7 @@ # features Settings File ################################################## import os +from schema import SCHEMA FEATURE_VERSION = os.environ["FEATURE_VERSION"] STEP_VERSION = os.environ["STEP_VERSION"] @@ -22,9 +23,12 @@ "PARAMS": { "bootstrap.servers": os.environ["CONSUMER_SERVER"], "group.id": os.environ["CONSUMER_GROUP_ID"], - "auto.offset.reset":"smallest", + "auto.offset.reset":"beginning", + "max.poll.interval.ms": 3600000, "enable.partition.eof": os.getenv("ENABLE_PARTITION_EOF", False), }, + "consume.timeout": int(os.getenv("CONSUME_TIMEOUT", 10)), + "consume.messages": int(os.getenv("CONSUME_MESSAGES", 1000)), } PRODUCER_CONFIG = { @@ -32,232 +36,7 @@ "PARAMS": { "bootstrap.servers": os.environ["PRODUCER_SERVER"], }, - "SCHEMA": { - "doc": "Features", - "name": "features_document", - "type": "record", - "fields": [ - {"name": "oid", "type": "string"}, - {"name": "candid", "type": "long"}, - { - "name": "features", - "type": { - "name": "features_record", - "type": "record", - "fields": [ - {"name": "Amplitude_1", "type": ["float", "null"]}, - {"name": "Amplitude_2", "type": ["float", "null"]}, - {"name": "AndersonDarling_1", "type": ["float", "null"]}, - {"name": "AndersonDarling_2", "type": ["float", "null"]}, - {"name": "Autocor_length_1", "type": ["double", "null"]}, - {"name": "Autocor_length_2", "type": ["double", "null"]}, - {"name": "Beyond1Std_1", "type": ["float", "null"]}, - {"name": "Beyond1Std_2", "type": ["float", "null"]}, - {"name": "Con_1", "type": ["double", "null"]}, - {"name": "Con_2", "type": ["double", "null"]}, - {"name": "Eta_e_1", "type": ["float", "null"]}, - {"name": "Eta_e_2", "type": ["float", "null"]}, - {"name": "ExcessVar_1", "type": ["double", "null"]}, - {"name": "ExcessVar_2", "type": ["double", "null"]}, - {"name": "GP_DRW_sigma_1", "type": ["double", "null"]}, - {"name": "GP_DRW_sigma_2", "type": ["double", "null"]}, - {"name": "GP_DRW_tau_1", "type": ["float", "null"]}, - {"name": "GP_DRW_tau_2", "type": ["float", "null"]}, - {"name": "Gskew_1", "type": ["float", "null"]}, - {"name": "Gskew_2", "type": ["float", "null"]}, - {"name": "Harmonics_mag_1_1", "type": ["float", "null"]}, - {"name": "Harmonics_mag_1_2", "type": ["float", "null"]}, - {"name": "Harmonics_mag_2_1", "type": ["float", "null"]}, - {"name": "Harmonics_mag_2_2", "type": ["float", "null"]}, - {"name": "Harmonics_mag_3_1", "type": ["float", "null"]}, - {"name": "Harmonics_mag_3_2", "type": ["float", "null"]}, - {"name": "Harmonics_mag_4_1", "type": ["float", "null"]}, - {"name": "Harmonics_mag_4_2", "type": ["float", "null"]}, - {"name": "Harmonics_mag_5_1", "type": ["float", "null"]}, - {"name": "Harmonics_mag_5_2", "type": ["float", "null"]}, - {"name": "Harmonics_mag_6_1", "type": ["float", "null"]}, - {"name": "Harmonics_mag_6_2", "type": ["float", "null"]}, - {"name": "Harmonics_mag_7_1", "type": ["float", "null"]}, - {"name": "Harmonics_mag_7_2", "type": ["float", "null"]}, - {"name": "Harmonics_mse_1", "type": ["double", "null"]}, - {"name": "Harmonics_mse_2", "type": ["double", "null"]}, - {"name": "Harmonics_phase_2_1", "type": ["float", "null"]}, - {"name": "Harmonics_phase_2_2", "type": ["float", "null"]}, - {"name": "Harmonics_phase_3_1", "type": ["float", "null"]}, - {"name": "Harmonics_phase_3_2", "type": ["float", "null"]}, - {"name": "Harmonics_phase_4_1", "type": ["float", "null"]}, - {"name": "Harmonics_phase_4_2", "type": ["float", "null"]}, - {"name": "Harmonics_phase_5_1", "type": ["float", "null"]}, - {"name": "Harmonics_phase_5_2", "type": ["float", "null"]}, - {"name": "Harmonics_phase_6_1", "type": ["float", "null"]}, - {"name": "Harmonics_phase_6_2", "type": ["float", "null"]}, - {"name": "Harmonics_phase_7_1", "type": ["float", "null"]}, - {"name": "Harmonics_phase_7_2", "type": ["float", "null"]}, - {"name": "IAR_phi_1", "type": ["double", "null"]}, - {"name": "IAR_phi_2", "type": ["float", "null"]}, - {"name": "LinearTrend_1", "type": ["float", "null"]}, - {"name": "LinearTrend_2", "type": ["double", "null"]}, - {"name": "MHPS_PN_flag_1", "type": ["double", "null"]}, - {"name": "MHPS_PN_flag_2", "type": ["double", "null"]}, - {"name": "MHPS_high_1", "type": ["float", "null"]}, - {"name": "MHPS_high_2", "type": ["double", "null"]}, - {"name": "MHPS_low_1", "type": ["float", "null"]}, - {"name": "MHPS_low_2", "type": ["float", "null"]}, - {"name": "MHPS_non_zero_1", "type": ["double", "null"]}, - {"name": "MHPS_non_zero_2", "type": ["double", "null"]}, - {"name": "MHPS_ratio_1", "type": ["float", "null"]}, - {"name": "MHPS_ratio_2", "type": ["float", "null"]}, - {"name": "MaxSlope_1", "type": ["float", "null"]}, - {"name": "MaxSlope_2", "type": ["float", "null"]}, - {"name": "Mean_1", "type": ["float", "null"]}, - {"name": "Mean_2", "type": ["float", "null"]}, - {"name": "Meanvariance_1", "type": ["float", "null"]}, - {"name": "Meanvariance_2", "type": ["float", "null"]}, - {"name": "MedianAbsDev_1", "type": ["float", "null"]}, - {"name": "MedianAbsDev_2", "type": ["float", "null"]}, - {"name": "MedianBRP_1", "type": ["float", "null"]}, - {"name": "MedianBRP_2", "type": ["float", "null"]}, - {"name": "Multiband_period", "type": ["float", "null"]}, - {"name": "PairSlopeTrend_1", "type": ["float", "null"]}, - {"name": "PairSlopeTrend_2", "type": ["float", "null"]}, - {"name": "PercentAmplitude_1", "type": ["float", "null"]}, - {"name": "PercentAmplitude_2", "type": ["float", "null"]}, - {"name": "Period_band_1", "type": ["float", "null"]}, - {"name": "Period_band_2", "type": ["float", "null"]}, - {"name": "delta_period_1", "type": ["float", "null"]}, - {"name": "delta_period_2", "type": ["float", "null"]}, - {"name": "Period_fit", "type": ["float", "null"]}, - {"name": "Power_rate_1/2", "type": ["float", "null"]}, - {"name": "Power_rate_1/3", "type": ["float", "null"]}, - {"name": "Power_rate_1/4", "type": ["float", "null"]}, - {"name": "Power_rate_2", "type": ["float", "null"]}, - {"name": "Power_rate_3", "type": ["float", "null"]}, - {"name": "Power_rate_4", "type": ["float", "null"]}, - {"name": "Psi_CS_1", "type": ["float", "null"]}, - {"name": "Psi_CS_2", "type": ["float", "null"]}, - {"name": "Psi_eta_1", "type": ["float", "null"]}, - {"name": "Psi_eta_2", "type": ["float", "null"]}, - {"name": "Pvar_1", "type": ["float", "null"]}, - {"name": "Pvar_2", "type": ["float", "null"]}, - {"name": "Q31_1", "type": ["float", "null"]}, - {"name": "Q31_2", "type": ["float", "null"]}, - {"name": "Rcs_1", "type": ["float", "null"]}, - {"name": "Rcs_2", "type": ["float", "null"]}, - {"name": "SF_ML_amplitude_1", "type": ["float", "null"]}, - {"name": "SF_ML_amplitude_2", "type": ["float", "null"]}, - {"name": "SF_ML_gamma_1", "type": ["float", "null"]}, - {"name": "SF_ML_gamma_2", "type": ["float", "null"]}, - {"name": "SPM_A_1", "type": ["float", "null"]}, - {"name": "SPM_A_2", "type": ["float", "null"]}, - {"name": "SPM_beta_1", "type": ["float", "null"]}, - {"name": "SPM_beta_2", "type": ["float", "null"]}, - {"name": "SPM_chi_1", "type": ["float", "null"]}, - {"name": "SPM_chi_2", "type": ["float", "null"]}, - {"name": "SPM_gamma_1", "type": ["float", "null"]}, - {"name": "SPM_gamma_2", "type": ["float", "null"]}, - {"name": "SPM_t0_1", "type": ["float", "null"]}, - {"name": "SPM_t0_2", "type": ["float", "null"]}, - {"name": "SPM_tau_fall_1", "type": ["float", "null"]}, - {"name": "SPM_tau_fall_2", "type": ["float", "null"]}, - {"name": "SPM_tau_rise_1", "type": ["float", "null"]}, - {"name": "SPM_tau_rise_2", "type": ["float", "null"]}, - {"name": "Skew_1", "type": ["float", "null"]}, - {"name": "Skew_2", "type": ["float", "null"]}, - {"name": "SmallKurtosis_1", "type": ["float", "null"]}, - {"name": "SmallKurtosis_2", "type": ["float", "null"]}, - {"name": "Std_1", "type": ["float", "null"]}, - {"name": "Std_2", "type": ["float", "null"]}, - {"name": "StetsonK_1", "type": ["float", "null"]}, - {"name": "StetsonK_2", "type": ["float", "null"]}, - {"name": "W1-W2", "type": ["double", "null"]}, - {"name": "W2-W3", "type": ["double", "null"]}, - {"name": "delta_mag_fid_1", "type": ["float", "null"]}, - {"name": "delta_mag_fid_2", "type": ["float", "null"]}, - {"name": "delta_mjd_fid_1", "type": ["float", "null"]}, - {"name": "delta_mjd_fid_2", "type": ["float", "null"]}, - {"name": "dmag_first_det_fid_1", "type": ["double", "null"]}, - {"name": "dmag_first_det_fid_2", "type": ["double", "null"]}, - {"name": "dmag_non_det_fid_1", "type": ["double", "null"]}, - {"name": "dmag_non_det_fid_2", "type": ["double", "null"]}, - {"name": "first_mag_1", "type": ["float", "null"]}, - {"name": "first_mag_2", "type": ["float", "null"]}, - {"name": "g-W2", "type": ["double", "null"]}, - {"name": "g-W3", "type": ["double", "null"]}, - {"name": "g-r_max", "type": ["float", "null"]}, - {"name": "g-r_max_corr", "type": ["float", "null"]}, - {"name": "g-r_mean", "type": ["float", "null"]}, - {"name": "g-r_mean_corr", "type": ["float", "null"]}, - {"name": "gal_b", "type": ["float", "null"]}, - {"name": "gal_l", "type": ["float", "null"]}, - {"name": "iqr_1", "type": ["float", "null"]}, - {"name": "iqr_2", "type": ["float", "null"]}, - { - "name": "last_diffmaglim_before_fid_1", - "type": ["double", "null"], - }, - { - "name": "last_diffmaglim_before_fid_2", - "type": ["double", "null"], - }, - {"name": "last_mjd_before_fid_1", "type": ["double", "null"]}, - {"name": "last_mjd_before_fid_2", "type": ["double", "null"]}, - { - "name": "max_diffmaglim_after_fid_1", - "type": ["double", "null"], - }, - { - "name": "max_diffmaglim_after_fid_2", - "type": ["double", "null"], - }, - { - "name": "max_diffmaglim_before_fid_1", - "type": ["double", "null"], - }, - { - "name": "max_diffmaglim_before_fid_2", - "type": ["double", "null"], - }, - {"name": "mean_mag_1", "type": ["float","null"]}, - {"name": "mean_mag_2", "type": ["float","null"]}, - { - "name": "median_diffmaglim_after_fid_1", - "type": ["double", "null"], - }, - { - "name": "median_diffmaglim_after_fid_2", - "type": ["double", "null"], - }, - { - "name": "median_diffmaglim_before_fid_1", - "type": ["double", "null"], - }, - { - "name": "median_diffmaglim_before_fid_2", - "type": ["double", "null"], - }, - {"name": "min_mag_1", "type": ["float", "null"]}, - {"name": "min_mag_2", "type": ["float", "null"]}, - {"name": "n_det_1", "type": ["double", "null"]}, - {"name": "n_det_2", "type": ["double", "null"]}, - {"name": "n_neg_1", "type": ["double", "null"]}, - {"name": "n_neg_2", "type": ["double", "null"]}, - {"name": "n_non_det_after_fid_1", "type": ["double", "null"]}, - {"name": "n_non_det_after_fid_2", "type": ["double", "null"]}, - {"name": "n_non_det_before_fid_1", "type": ["double", "null"]}, - {"name": "n_non_det_before_fid_2", "type": ["double", "null"]}, - {"name": "n_pos_1", "type": ["double", "null"]}, - {"name": "n_pos_2", "type": ["double", "null"]}, - {"name": "positive_fraction_1", "type": ["double", "null"]}, - {"name": "positive_fraction_2", "type": ["double", "null"]}, - {"name": "r-W2", "type": ["double", "null"]}, - {"name": "r-W3", "type": ["double", "null"]}, - {"name": "rb", "type": ["float", "null"]}, - {"name": "sgscore1", "type": ["float", "null"]} - ], - }, - }, - ], - }, + "SCHEMA": SCHEMA } METRICS_CONFIG = { diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..facdf92 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,87 @@ +import pytest +from confluent_kafka.admin import AdminClient, NewTopic +from confluent_kafka import Producer, Consumer +import glob +import os +import psycopg2 + + +FILE_PATH = os.path.dirname(os.path.abspath(__file__)) +EXAMPLES_PATH = os.path.abspath(os.path.join(FILE_PATH, "../examples/avro_test")) + + +@pytest.fixture(scope="session") +def docker_compose_file(pytestconfig): + return os.path.join( + str(pytestconfig.rootdir), "tests/integration", "docker-compose.yml" + ) + + +def read_avro(): + files = glob.glob(os.path.join(EXAMPLES_PATH, "*.avro")) + files.sort() + nfiles = len(files) + for f in files: + with open(f, "rb") as fo: + yield fo.read() + + +def is_responsive_kafka(url): + client = AdminClient({"bootstrap.servers": url}) + topics = ["test"] + new_topics = [NewTopic(topic, num_partitions=1) for topic in topics] + fs = client.create_topics(new_topics) + for topic, f in fs.items(): + try: + f.result() + return True + except Exception as e: + return False + + +@pytest.fixture(scope="session") +def kafka_service(docker_ip, docker_services): + """Ensure that Kafka service is up and responsive.""" + print("Kafka", docker_ip) + topics = ["test"] + # `port_for` takes a container port and returns the corresponding host port + port = docker_services.port_for("kafka", 9094) + server = "{}:{}".format(docker_ip, port) + docker_services.wait_until_responsive( + timeout=30.0, pause=0.1, check=lambda: is_responsive_kafka(server) + ) + config = {"bootstrap.servers": "localhost:9094"} + producer = Producer(config) + try: + for topic in topics: + for data in read_avro(): + producer.produce(topic, value=data) + producer.flush() + print(f"produced to {topic}") + except Exception as e: + print(f"failed to produce to topic {topic}: {e}") + return server + + +def is_responsive_psql(url): + try: + conn = psycopg2.connect( + f"dbname='postgres' user='postgres' host=localhost password='postgres'" + ) + conn.close() + return True + except: + return False + + +@pytest.fixture(scope="session") +def psql_service(docker_ip, docker_services): + """Ensure that Kafka service is up and responsive.""" + # `port_for` takes a container port and returns the corresponding host port + port = docker_services.port_for("postgres", 5432) + server = "{}:{}".format(docker_ip, port) + print("psql", server) + docker_services.wait_until_responsive( + timeout=30.0, pause=0.1, check=lambda: is_responsive_psql(server) + ) + return server diff --git a/tests/examples/avro_test/1000151433015015013.avro b/tests/examples/avro_test/1000151433015015013.avro new file mode 100644 index 0000000..d631eae --- /dev/null +++ b/tests/examples/avro_test/1000151433015015013.avro @@ -0,0 +1,205 @@ +Objavro.codecnullavro.schema{"type": "record", "version": "3.3", "name": "alert", "namespace": "ztf", "fields": [{"type": "string", "name": "schemavsn", "doc": "schema version used"}, {"type": "string", "name": "publisher", "doc": "origin of alert packet"}, {"type": "string", "name": "objectId", "doc": "object identifier or name"}, {"type": "long", "name": "candid"}, {"type": {"type": "record", "version": "3.3", "name": "candidate", "namespace": "ztf.alert", "fields": [{"type": "double", "name": "jd", "doc": "Observation Julian date at start of exposure [days]"}, {"type": "int", "name": "fid", "doc": "Filter ID (1=g; 2=R; 3=i)"}, {"type": "long", "name": "pid", "doc": "Processing ID for science image to facilitate archive retrieval"}, {"type": ["float", "null"], "name": "diffmaglim", "default": null, "doc": "Expected 5-sigma mag limit in difference image based on global noise estimate [mag]"}, {"type": ["string", "null"], "name": "pdiffimfilename", "default": null, "doc": "filename of positive (sci minus ref) difference image"}, {"type": ["string", "null"], "name": "programpi", "default": null, "doc": "Principal investigator attached to program ID"}, {"type": "int", "name": "programid", "doc": "Program ID: encodes either public, collab, or caltech mode"}, {"type": "long", "name": "candid", "doc": "Candidate ID from operations DB"}, {"type": "string", "name": "isdiffpos", "doc": "t or 1 => candidate is from positive (sci minus ref) subtraction; f or 0 => candidate is from negative (ref minus sci) subtraction"}, {"type": ["long", "null"], "name": "tblid", "default": null, "doc": "Internal pipeline table extraction ID"}, {"type": ["int", "null"], "name": "nid", "default": null, "doc": "Night ID"}, {"type": ["int", "null"], "name": "rcid", "default": null, "doc": "Readout channel ID [00 .. 63]"}, {"type": ["int", "null"], "name": "field", "default": null, "doc": "ZTF field ID"}, {"type": ["float", "null"], "name": "xpos", "default": null, "doc": "x-image position of candidate [pixels]"}, {"type": ["float", "null"], "name": "ypos", "default": null, "doc": "y-image position of candidate [pixels]"}, {"type": "double", "name": "ra", "doc": "Right Ascension of candidate; J2000 [deg]"}, {"type": "double", "name": "dec", "doc": "Declination of candidate; J2000 [deg]"}, {"type": "float", "name": "magpsf", "doc": "Magnitude from PSF-fit photometry [mag]"}, {"type": "float", "name": "sigmapsf", "doc": "1-sigma uncertainty in magpsf [mag]"}, {"type": ["float", "null"], "name": "chipsf", "default": null, "doc": "Reduced chi-square for PSF-fit"}, {"type": ["float", "null"], "name": "magap", "default": null, "doc": "Aperture mag using 14 pixel diameter aperture [mag]"}, {"type": ["float", "null"], "name": "sigmagap", "default": null, "doc": "1-sigma uncertainty in magap [mag]"}, {"type": ["float", "null"], "name": "distnr", "default": null, "doc": "distance to nearest source in reference image PSF-catalog [pixels]"}, {"type": ["float", "null"], "name": "magnr", "default": null, "doc": "magnitude of nearest source in reference image PSF-catalog [mag]"}, {"type": ["float", "null"], "name": "sigmagnr", "default": null, "doc": "1-sigma uncertainty in magnr [mag]"}, {"type": ["float", "null"], "name": "chinr", "default": null, "doc": "DAOPhot chi parameter of nearest source in reference image PSF-catalog"}, {"type": ["float", "null"], "name": "sharpnr", "default": null, "doc": "DAOPhot sharp parameter of nearest source in reference image PSF-catalog"}, {"type": ["float", "null"], "name": "sky", "default": null, "doc": "Local sky background estimate [DN]"}, {"type": ["float", "null"], "name": "magdiff", "default": null, "doc": "Difference: magap - magpsf [mag]"}, {"type": ["float", "null"], "name": "fwhm", "default": null, "doc": "Full Width Half Max assuming a Gaussian core, from SExtractor [pixels]"}, {"type": ["float", "null"], "name": "classtar", "default": null, "doc": "Star/Galaxy classification score from SExtractor"}, {"type": ["float", "null"], "name": "mindtoedge", "default": null, "doc": "Distance to nearest edge in image [pixels]"}, {"type": ["float", "null"], "name": "magfromlim", "default": null, "doc": "Difference: diffmaglim - magap [mag]"}, {"type": ["float", "null"], "name": "seeratio", "default": null, "doc": "Ratio: difffwhm / fwhm"}, {"type": ["float", "null"], "name": "aimage", "default": null, "doc": "Windowed profile RMS afloat major axis from SExtractor [pixels]"}, {"type": ["float", "null"], "name": "bimage", "default": null, "doc": "Windowed profile RMS afloat minor axis from SExtractor [pixels]"}, {"type": ["float", "null"], "name": "aimagerat", "default": null, "doc": "Ratio: aimage / fwhm"}, {"type": ["float", "null"], "name": "bimagerat", "default": null, "doc": "Ratio: bimage / fwhm"}, {"type": ["float", "null"], "name": "elong", "default": null, "doc": "Ratio: aimage / bimage"}, {"type": ["int", "null"], "name": "nneg", "default": null, "doc": "number of negative pixels in a 5 x 5 pixel stamp"}, {"type": ["int", "null"], "name": "nbad", "default": null, "doc": "number of prior-tagged bad pixels in a 5 x 5 pixel stamp"}, {"type": ["float", "null"], "name": "rb", "default": null, "doc": "RealBogus quality score from Random Forest classifier; range is 0 to 1 where closer to 1 is more reliable"}, {"type": ["float", "null"], "name": "ssdistnr", "default": null, "doc": "distance to nearest known solar system object if exists within 30 arcsec [arcsec]"}, {"type": ["float", "null"], "name": "ssmagnr", "default": null, "doc": "magnitude of nearest known solar system object if exists within 30 arcsec (usually V-band from MPC archive) [mag]"}, {"type": ["string", "null"], "name": "ssnamenr", "default": null, "doc": "name of nearest known solar system object if exists within 30 arcsec (from MPC archive)"}, {"type": ["float", "null"], "name": "sumrat", "default": null, "doc": "Ratio: sum(pixels) / sum(|pixels|) in a 5 x 5 pixel stamp where stamp is first median-filtered to mitigate outliers"}, {"type": ["float", "null"], "name": "magapbig", "default": null, "doc": "Aperture mag using 18 pixel diameter aperture [mag]"}, {"type": ["float", "null"], "name": "sigmagapbig", "default": null, "doc": "1-sigma uncertainty in magapbig [mag]"}, {"type": "double", "name": "ranr", "doc": "Right Ascension of nearest source in reference image PSF-catalog; J2000 [deg]"}, {"type": "double", "name": "decnr", "doc": "Declination of nearest source in reference image PSF-catalog; J2000 [deg]"}, {"type": ["float", "null"], "name": "sgmag1", "default": null, "doc": "g-band PSF-fit magnitude of closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, {"type": ["float", "null"], "name": "srmag1", "default": null, "doc": "r-band PSF-fit magnitude of closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, {"type": ["float", "null"], "name": "simag1", "default": null, "doc": "i-band PSF-fit magnitude of closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, {"type": ["float", "null"], "name": "szmag1", "default": null, "doc": "z-band PSF-fit magnitude of closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, {"type": ["float", "null"], "name": "sgscore1", "default": null, "doc": "Star/Galaxy score of closest source from PS1 catalog; if exists within 30 arcsec: 0 <= sgscore <= 1 where closer to 1 implies higher likelihood of being a star"}, {"type": ["float", "null"], "name": "distpsnr1", "default": null, "doc": "Distance to closest source from PS1 catalog; if exists within 30 arcsec [arcsec]"}, {"type": "int", "name": "ndethist", "doc": "Number of spatially-coincident detections falling within 1.5 arcsec going back to beginning of survey; only detections that fell on the same field and readout-channel ID where the input candidate was observed are counted"}, {"type": "int", "name": "ncovhist", "doc": "Number of times input candidate position fell on any field and readout-channel going back to beginning of survey"}, {"type": ["double", "null"], "name": "jdstarthist", "default": null, "doc": "Earliest Julian date of epoch corresponding to ndethist [days]"}, {"type": ["double", "null"], "name": "jdendhist", "default": null, "doc": "Latest Julian date of epoch corresponding to ndethist [days]"}, {"type": ["double", "null"], "name": "scorr", "default": null, "doc": "Peak-pixel signal-to-noise ratio in point source matched-filtered detection image"}, {"type": ["int", "null"], "name": "tooflag", "default": 0, "doc": "1 => candidate is from a Target-of-Opportunity (ToO) exposure; 0 => candidate is from a non-ToO exposure"}, {"type": ["long", "null"], "name": "objectidps1", "default": null, "doc": "Object ID of closest source from PS1 catalog; if exists within 30 arcsec"}, {"type": ["long", "null"], "name": "objectidps2", "default": null, "doc": "Object ID of second closest source from PS1 catalog; if exists within 30 arcsec"}, {"type": ["float", "null"], "name": "sgmag2", "default": null, "doc": "g-band PSF-fit magnitude of second closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, {"type": ["float", "null"], "name": "srmag2", "default": null, "doc": "r-band PSF-fit magnitude of second closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, {"type": ["float", "null"], "name": "simag2", "default": null, "doc": "i-band PSF-fit magnitude of second closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, {"type": ["float", "null"], "name": "szmag2", "default": null, "doc": "z-band PSF-fit magnitude of second closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, {"type": ["float", "null"], "name": "sgscore2", "default": null, "doc": "Star/Galaxy score of second closest source from PS1 catalog; if exists within 30 arcsec: 0 <= sgscore <= 1 where closer to 1 implies higher likelihood of being a star"}, {"type": ["float", "null"], "name": "distpsnr2", "default": null, "doc": "Distance to second closest source from PS1 catalog; if exists within 30 arcsec [arcsec]"}, {"type": ["long", "null"], "name": "objectidps3", "default": null, "doc": "Object ID of third closest source from PS1 catalog; if exists within 30 arcsec"}, {"type": ["float", "null"], "name": "sgmag3", "default": null, "doc": "g-band PSF-fit magnitude of third closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, {"type": ["float", "null"], "name": "srmag3", "default": null, "doc": "r-band PSF-fit magnitude of third closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, {"type": ["float", "null"], "name": "simag3", "default": null, "doc": "i-band PSF-fit magnitude of third closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, {"type": ["float", "null"], "name": "szmag3", "default": null, "doc": "z-band PSF-fit magnitude of third closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, {"type": ["float", "null"], "name": "sgscore3", "default": null, "doc": "Star/Galaxy score of third closest source from PS1 catalog; if exists within 30 arcsec: 0 <= sgscore <= 1 where closer to 1 implies higher likelihood of being a star"}, {"type": ["float", "null"], "name": "distpsnr3", "default": null, "doc": "Distance to third closest source from PS1 catalog; if exists within 30 arcsec [arcsec]"}, {"type": "int", "name": "nmtchps", "doc": "Number of source matches from PS1 catalog falling within 30 arcsec"}, {"type": "long", "name": "rfid", "doc": "Processing ID for reference image to facilitate archive retrieval"}, {"type": "double", "name": "jdstartref", "doc": "Observation Julian date of earliest exposure used to generate reference image [days]"}, {"type": "double", "name": "jdendref", "doc": "Observation Julian date of latest exposure used to generate reference image [days]"}, {"type": "int", "name": "nframesref", "doc": "Number of frames (epochal images) used to generate reference image"}, {"type": "string", "name": "rbversion", "doc": "version of Random Forest classifier model used to assign RealBogus (rb) quality score"}, {"type": ["float", "null"], "name": "dsnrms", "default": null, "doc": "Ratio: D/stddev(D) on event position where D = difference image"}, {"type": ["float", "null"], "name": "ssnrms", "default": null, "doc": "Ratio: S/stddev(S) on event position where S = image of convolution: D (x) PSF(D)"}, {"type": ["float", "null"], "name": "dsdiff", "default": null, "doc": "Difference of statistics: dsnrms - ssnrms"}, {"type": ["float", "null"], "name": "magzpsci", "default": null, "doc": "Magnitude zero point for photometry estimates [mag]"}, {"type": ["float", "null"], "name": "magzpsciunc", "default": null, "doc": "Magnitude zero point uncertainty (in magzpsci) [mag]"}, {"type": ["float", "null"], "name": "magzpscirms", "default": null, "doc": "RMS (deviation from average) in all differences between instrumental photometry and matched photometric calibrators from science image processing [mag]"}, {"type": "int", "name": "nmatches", "doc": "Number of PS1 photometric calibrators used to calibrate science image from science image processing"}, {"type": ["float", "null"], "name": "clrcoeff", "default": null, "doc": "Color coefficient from linear fit from photometric calibration of science image"}, {"type": ["float", "null"], "name": "clrcounc", "default": null, "doc": "Color coefficient uncertainty from linear fit (corresponding to clrcoeff)"}, {"type": ["float", "null"], "name": "zpclrcov", "default": null, "doc": "Covariance in magzpsci and clrcoeff from science image processing [mag^2]"}, {"type": ["float", "null"], "name": "zpmed", "default": null, "doc": "Magnitude zero point from median of all differences between instrumental photometry and matched photometric calibrators from science image processing [mag]"}, {"type": ["float", "null"], "name": "clrmed", "default": null, "doc": "Median color of all PS1 photometric calibrators used from science image processing [mag]: for filter (fid) = 1, 2, 3, PS1 color used = g-r, g-r, r-i respectively"}, {"type": ["float", "null"], "name": "clrrms", "default": null, "doc": "RMS color (deviation from average) of all PS1 photometric calibrators used from science image processing [mag]"}, {"type": ["float", "null"], "name": "neargaia", "default": null, "doc": "Distance to closest source from Gaia DR1 catalog irrespective of magnitude; if exists within 90 arcsec [arcsec]"}, {"type": ["float", "null"], "name": "neargaiabright", "default": null, "doc": "Distance to closest source from Gaia DR1 catalog brighter than magnitude 14; if exists within 90 arcsec [arcsec]"}, {"type": ["float", "null"], "name": "maggaia", "default": null, "doc": "Gaia (G-band) magnitude of closest source from Gaia DR1 catalog irrespective of magnitude; if exists within 90 arcsec [mag]"}, {"type": ["float", "null"], "name": "maggaiabright", "default": null, "doc": "Gaia (G-band) magnitude of closest source from Gaia DR1 catalog brighter than magnitude 14; if exists within 90 arcsec [mag]"}, {"type": ["float", "null"], "name": "exptime", "default": null, "doc": "Integration time of camera exposure [sec]"}, {"type": ["float", "null"], "name": "drb", "default": null, "doc": "RealBogus quality score from Deep-Learning-based classifier; range is 0 to 1 where closer to 1 is more reliable"}, {"type": "string", "name": "drbversion", "doc": "version of Deep-Learning-based classifier model used to assign RealBogus (drb) quality score"}], "doc": "avro alert schema"}, "name": "candidate"}, {"type": [{"type": "array", "items": {"type": "record", "version": "3.3", "name": "prv_candidate", "namespace": "ztf.alert", "fields": [{"type": "double", "name": "jd", "doc": "Observation Julian date at start of exposure [days]"}, {"type": "int", "name": "fid", "doc": "Filter ID (1=g; 2=R; 3=i)"}, {"type": "long", "name": "pid", "doc": "Processing ID for image"}, {"type": ["float", "null"], "name": "diffmaglim", "default": null, "doc": "Expected 5-sigma mag limit in difference image based on global noise estimate [mag]"}, {"type": ["string", "null"], "name": "pdiffimfilename", "default": null, "doc": "filename of positive (sci minus ref) difference image"}, {"type": ["string", "null"], "name": "programpi", "default": null, "doc": "Principal investigator attached to program ID"}, {"type": "int", "name": "programid", "doc": "Program ID: encodes either public, collab, or caltech mode"}, {"type": ["long", "null"], "name": "candid", "doc": "Candidate ID from operations DB"}, {"type": ["string", "null"], "name": "isdiffpos", "doc": "t or 1 => candidate is from positive (sci minus ref) subtraction; f or 0 => candidate is from negative (ref minus sci) subtraction"}, {"type": ["long", "null"], "name": "tblid", "default": null, "doc": "Internal pipeline table extraction ID"}, {"type": ["int", "null"], "name": "nid", "default": null, "doc": "Night ID"}, {"type": ["int", "null"], "name": "rcid", "default": null, "doc": "Readout channel ID [00 .. 63]"}, {"type": ["int", "null"], "name": "field", "default": null, "doc": "ZTF field ID"}, {"type": ["float", "null"], "name": "xpos", "default": null, "doc": "x-image position of candidate [pixels]"}, {"type": ["float", "null"], "name": "ypos", "default": null, "doc": "y-image position of candidate [pixels]"}, {"type": ["double", "null"], "name": "ra", "doc": "Right Ascension of candidate; J2000 [deg]"}, {"type": ["double", "null"], "name": "dec", "doc": "Declination of candidate; J2000 [deg]"}, {"type": ["float", "null"], "name": "magpsf", "doc": "Magnitude from PSF-fit photometry [mag]"}, {"type": ["float", "null"], "name": "sigmapsf", "doc": "1-sigma uncertainty in magpsf [mag]"}, {"type": ["float", "null"], "name": "chipsf", "default": null, "doc": "Reduced chi-square for PSF-fit"}, {"type": ["float", "null"], "name": "magap", "default": null, "doc": "Aperture mag using 14 pixel diameter aperture [mag]"}, {"type": ["float", "null"], "name": "sigmagap", "default": null, "doc": "1-sigma uncertainty in magap [mag]"}, {"type": ["float", "null"], "name": "distnr", "default": null, "doc": "distance to nearest source in reference image PSF-catalog [pixels]"}, {"type": ["float", "null"], "name": "magnr", "default": null, "doc": "magnitude of nearest source in reference image PSF-catalog [mag]"}, {"type": ["float", "null"], "name": "sigmagnr", "default": null, "doc": "1-sigma uncertainty in magnr [mag]"}, {"type": ["float", "null"], "name": "chinr", "default": null, "doc": "DAOPhot chi parameter of nearest source in reference image PSF-catalog"}, {"type": ["float", "null"], "name": "sharpnr", "default": null, "doc": "DAOPhot sharp parameter of nearest source in reference image PSF-catalog"}, {"type": ["float", "null"], "name": "sky", "default": null, "doc": "Local sky background estimate [DN]"}, {"type": ["float", "null"], "name": "magdiff", "default": null, "doc": "Difference: magap - magpsf [mag]"}, {"type": ["float", "null"], "name": "fwhm", "default": null, "doc": "Full Width Half Max assuming a Gaussian core, from SExtractor [pixels]"}, {"type": ["float", "null"], "name": "classtar", "default": null, "doc": "Star/Galaxy classification score from SExtractor"}, {"type": ["float", "null"], "name": "mindtoedge", "default": null, "doc": "Distance to nearest edge in image [pixels]"}, {"type": ["float", "null"], "name": "magfromlim", "default": null, "doc": "Difference: diffmaglim - magap [mag]"}, {"type": ["float", "null"], "name": "seeratio", "default": null, "doc": "Ratio: difffwhm / fwhm"}, {"type": ["float", "null"], "name": "aimage", "default": null, "doc": "Windowed profile RMS afloat major axis from SExtractor [pixels]"}, {"type": ["float", "null"], "name": "bimage", "default": null, "doc": "Windowed profile RMS afloat minor axis from SExtractor [pixels]"}, {"type": ["float", "null"], "name": "aimagerat", "default": null, "doc": "Ratio: aimage / fwhm"}, {"type": ["float", "null"], "name": "bimagerat", "default": null, "doc": "Ratio: bimage / fwhm"}, {"type": ["float", "null"], "name": "elong", "default": null, "doc": "Ratio: aimage / bimage"}, {"type": ["int", "null"], "name": "nneg", "default": null, "doc": "number of negative pixels in a 5 x 5 pixel stamp"}, {"type": ["int", "null"], "name": "nbad", "default": null, "doc": "number of prior-tagged bad pixels in a 5 x 5 pixel stamp"}, {"type": ["float", "null"], "name": "rb", "default": null, "doc": "RealBogus quality score; range is 0 to 1 where closer to 1 is more reliable"}, {"type": ["float", "null"], "name": "ssdistnr", "default": null, "doc": "distance to nearest known solar system object if exists within 30 arcsec [arcsec]"}, {"type": ["float", "null"], "name": "ssmagnr", "default": null, "doc": "magnitude of nearest known solar system object if exists within 30 arcsec (usually V-band from MPC archive) [mag]"}, {"type": ["string", "null"], "name": "ssnamenr", "default": null, "doc": "name of nearest known solar system object if exists within 30 arcsec (from MPC archive)"}, {"type": ["float", "null"], "name": "sumrat", "default": null, "doc": "Ratio: sum(pixels) / sum(|pixels|) in a 5 x 5 pixel stamp where stamp is first median-filtered to mitigate outliers"}, {"type": ["float", "null"], "name": "magapbig", "default": null, "doc": "Aperture mag using 18 pixel diameter aperture [mag]"}, {"type": ["float", "null"], "name": "sigmagapbig", "default": null, "doc": "1-sigma uncertainty in magapbig [mag]"}, {"type": ["double", "null"], "name": "ranr", "doc": "Right Ascension of nearest source in reference image PSF-catalog; J2000 [deg]"}, {"type": ["double", "null"], "name": "decnr", "doc": "Declination of nearest source in reference image PSF-catalog; J2000 [deg]"}, {"type": ["double", "null"], "name": "scorr", "default": null, "doc": "Peak-pixel signal-to-noise ratio in point source matched-filtered detection image"}, {"type": ["float", "null"], "name": "magzpsci", "default": null, "doc": "Magnitude zero point for photometry estimates [mag]"}, {"type": ["float", "null"], "name": "magzpsciunc", "default": null, "doc": "Magnitude zero point uncertainty (in magzpsci) [mag]"}, {"type": ["float", "null"], "name": "magzpscirms", "default": null, "doc": "RMS (deviation from average) in all differences between instrumental photometry and matched photometric calibrators from science image processing [mag]"}, {"type": ["float", "null"], "name": "clrcoeff", "default": null, "doc": "Color coefficient from linear fit from photometric calibration of science image"}, {"type": ["float", "null"], "name": "clrcounc", "default": null, "doc": "Color coefficient uncertainty from linear fit (corresponding to clrcoeff)"}, {"type": "string", "name": "rbversion", "doc": "version of RealBogus model/classifier used to assign rb quality score"}], "doc": "avro alert schema"}}, "null"], "name": "prv_candidates", "default": null}, {"type": [{"type": "record", "version": "3.3", "name": "cutout", "namespace": "ztf.alert", "fields": [{"type": "string", "name": "fileName"}, {"type": "bytes", "name": "stampData", "doc": "fits.gz"}], "doc": "avro alert schema"}, "null"], "name": "cutoutScience", "default": null}, {"type": ["ztf.alert.cutout", "null"], "name": "cutoutTemplate", "default": null}, {"type": ["ztf.alert.cutout", "null"], "name": "cutoutDifference", "default": null}], "doc": "avro alert schema for ZTF (www.ztf.caltech.edu)"}挈;Р{إ3.32ZTF (www.ztf.caltech.edu)ZTF19aaapkto٩缛:bSABAڛ:Arztf_20190928151377_000796_zr_c08_o_q3_scimrefdiffimg.fitsTESS٩缛t< jzDFDglo@S*?@!Az>>nAt=ff?/=>Kff?|?jzDy?n?*??>S>>]?}v?yynullڧi?|A(\>,o@dy1J@pA6AVA]Aǁ?pO>+k[^BA:bSABA߽/"@ꞎԬꞎA,ԟAAA&@ꞎyǰAAXA=TMmAҞӀ%BAp^}5BAt17_f5_c3B8A䮧A%MAہ6a= =[_6-Am?>r>!3-BiAOPAAy? +d6_m7b5d2BA8xA/ztf/archive/sci/2019/0829/288576/ztf_20190829288576_000795_zr_c05_o_q4_scimrefdiffimg.fits.fzTESSt& q CRHDqo@ɶª0J@w>A$>F?J{A >?>!A<ˡ?=AIa)\@-}?q C??w_?SC?>ff>?g|???AKG>A?)A䃞>>nAt=ff?/=+=3n>?5^z?uD!>? 0?q= +?h>r>?33s?Q0w?A'>,o@dy1J@m4%@ +hAw06<0=x +7t17_f5_c3g2BAԱ83A/ztf/archive/sci/2019/0829/311713/ztf_20190829311713_000795_zg_c05_o_q4_scimrefdiffimg.fits.fzTESSt& ^CD'q~o@uPx3J@AQ >\>Aj<?T<ù>9{W|@ p?^CԘ\?|4?z?!r?$(>u>?k??AW/>`Co@L/1J@Sr3#@A!17k =YW7t17_f5_c3h2BA8\A/ztf/archive/sci/2019/0829/312488/ztf_20190829312488_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESSt< rD3DHh˹o@#1J@?A3>@bAsR>1#>DA㥛<G?)\=6=U>!@z?rD<?|,?z4?o#?>->?Ew?8Ll?fA[>l@o@daU1J@b=9+@Ap7'=Mɂ7t17_f5_c32BA84A/ztf/archive/sci/2019/0830/262523/ztf_20190830262523_000795_zr_c05_o_q4_scimrefdiffimg.fits.fzTESSوt4& CD67Kno@nZ:0J@A=Y?hAVm>vk9>!A<ˡ?='f>m>z?v?C՗)???X9?M>B>f? +Lt??iA@>{@ԚA?>>nAt=ff?/=Ӥ4>?=?w?tDt??\"??>1>?t??֙A,eY>,o@dy1J@>$@Mm!@AF̹6<k=7t17_f5_c3V|2BA8)A/ztf/archive/sci/2019/0830/287928/ztf_20190830287928_000795_zg_c05_o_q4_scimrefdiffimg.fits.fzTESSݔt$& CD4Ro@0J@YA<>W @IAr>8=Aj<?T<9^>k={@z?Cy?#?C?#?$^>t> +? l?fn?AH?>`Co@L/1J@Dl)+@0Au6xa<GdNE7t17_f5_c32BA8A/ztf/archive/sci/2019/0830/288796/ztf_20190830288796_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESSt< hwDD 1o@/J@ߙA#>8@גA>">DA㥛<G?)\=A>YffF@r?hwDr?]?&?|?= +W>M>? +j?o~g?A>yX>l@o@daU1J@+.+@Af7=0v7t17_f5_c3a3BA֩8[SA/ztf/archive/sci/2019/0831/263970/ztf_20190831263970_000795_zr_c05_o_q4_scimrefdiffimg.fits.fzTESS἖t& hCD-)o@v\C_0J@A;93>m?6oF=>!A<ˡ?=H> :@y?hCXb?ʌ?n?-2?>>f? 0Fp?h?֘Av>>ۊ?AV>>nAt=ff?/=S><=?R~?yD-6??OM??>->? Su? y?Ax>,o@dy1J@aM#@A6<= 7t17_f5_c3}e3BAԏ8/A/ztf/archive/sci/2019/0831/289306/ztf_20190831289306_000795_zg_c05_o_q4_scimrefdiffimg.fits.fzTESS䑢t& CDKo@'I2J@GA/m>?OA'>>Aj<?T<:?PV=G!@Hz?C[?.?)\O?Nb0?X>>z?w??AZ>`Co@L/1J@8EGr&@A`7<uMrNr7t17_f5_c3"e3BA8ΡA/ztf/archive/sci/2019/0831/290093/ztf_20190831290093_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESSęΚt< jvDDb|=o@F0J@ؙA>t? A!4>v>DA㥛<G?)\=wۅ>= +@z?jvD4X?Ҍ?9?h-?Xp>>~?9w??sיA'1H>l@o@daU1J@V'@^:AL6b<bo̪I7t17_f5_c3Ʊ3BA8A/ztf/archive/sci/2019/0901/287894/ztf_20190901287894_000795_zr_c05_o_q4_scimrefdiffimg.fits.fzTESSt& h!CzD{eo@.J@0A>?E؛ASc>o>!A<ˡ?=Pp>>\@bx?h!CO(?㍘?z4?V ?>IJ>Tƣ?9w?=}h?&A>3BAΐ8A/ztf/archive/sci/2019/0901/288681/ztf_20190901288681_000796_zr_c08_o_q3_scimrefdiffimg.fits.fzTESSt< wDDto@ .J@J{A4>lC?0A0Lf>u>nAt=ff?/=G=3m< ?3?wD0&?@:?{??h>"?[@l?tC{?!Ak>,o@dy1J@X2ı'@5AZ6 <%v=Q,7t17_f5_c3j3BA֗8iA/ztf/archive/sci/2019/0901/311262/ztf_20190901311262_000795_zg_c05_o_q4_scimrefdiffimg.fits.fzTESS꽍t & CqD\W o@({1J@A+>?J A78>z>Aj<?T<f>S[*=,@y?CN?>y?e? 0?ᷩ>w>肦?o??Ah"l>`Co@L/1J@/'+%@)AF7=V97t17_f5_c3xX3BA8A/ztf/archive/sci/2019/0901/312049/ztf_20190901312049_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESS餻؋t< qDͤD;ro@<0J@fA=)? ҘA'>=?DA㥛<G?)\=a;@y?qD`x?!?/=?2?Rc>L>R?x?x{|?†A)M>l@o@daU1J@6[ $@KA579=57t17_f5_c3_a5BA8/]A/ztf/archive/sci/2019/0904/264225/ztf_20190904264225_000795_zr_c05_o_q4_scimrefdiffimg.fits.fzTESSɱt& NbCD]}o@Ul0J@gAjm>F?AΈr?>!A<ˡ?=6Ҿ9U?ϲ?Zd?NbC;@v?Nb?e>>{?(?^c?cAd?? A>Gr!?nAt=ff?/=>p>q= +@Qx?^wD=$?z?Ck?Օ>I>?p=??FA->,o@dy1J@"~"@A7m==MA8t17_f5_c3V|d5BA8hA/ztf/archive/sci/2019/0904/288009/ztf_20190904288009_000795_zg_c05_o_q4_scimrefdiffimg.fits.fzTESSݘļt +& aCDLo@:#01J@!0A>C?!A2>[/=Aj<?T<E> @bx?aCI>?ђ?jt?nR?>yX>ݔ?6 v??{Al>`Co@L/1J@гY'@ݤAMB7<=w=7t17_f5_c3d5BA8 ҝA/ztf/archive/sci/2019/0904/288785/ztf_20190904288785_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESS༌t < LuD*DIo@́/J@yADL>.s?EAm4>*?DA㥛<G?)\= +d2 @v?LuD1?$?n?1L?>9>ו?_q??YAh>l@o@daU1J@H%@ŠAh7=k<"T7t17_f5_c3~a6BA8X9A/ztf/archive/sci/2019/0906/262859/ztf_20190906262859_000795_zr_c05_o_q4_scimrefdiffimg.fits.fzTESSӘٺt & CD¡xo@pWTi0J@sA$=j?eA;>*>!A<ˡ?=>`\@bx?C;?4?-?&?fj>oc>Z?]?}A?3ĘA!=>?A>>nAt=ff?/=rO=>(?5^z?uD^>Y?#??6>>J?Qx??A>,o@dy1J@\tYL@{Aλ6{<b=7t17_f5_c3.d6BA8CA/ztf/archive/sci/2019/0906/286285/ztf_20190906286285_000795_zg_c05_o_q4_scimrefdiffimg.fits.fzTESSĝɓt& |C)lDxo@ʢU/J@ ƙAE/>,e?OAˡ>6>Aj<?T<=?n)\@z?|C Ҕ???+G?sh1?Ա>ff>S?( y??zAˡE>`Co@L/1J@q -*@uA. 7q<!n7t17_f5_c3d6BA8?A/ztf/archive/sci/2019/0906/287072/ztf_20190906287072_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESS¹ɓt< )tDuD o@Qk0J@2fAS>L @3Aף0>>DA㥛<G?)\=36>= @Zd{?)tD=ea??1,?\"?ӝ>-#>du?9w?w?țA]܆>l@o@daU1J@?ܵ\,@8A<7<`$i7t17_f5_c3K86BA9ʲA/ztf/archive/sci/2019/0907/262465/ztf_20190907262465_000795_zr_c05_o_q4_scimrefdiffimg.fits.fzTESSˆt& =CqD*qo@/J@ྙA=g@A%>og>!A<ˡ?=&:>{ף?|?=C`?U?;/?"?>e5>C? +z?x?= +A2>?'A/>'X>nAt=ff?/=ɰ.?憾8@Ev?rD+?F?n2? ?v>t)^>8J?k?Y?mŘA ~>,o@dy1J@ "@Aā6<=7t17_f5_c36BA럀9A/ztf/archive/sci/2019/0907/285613/ztf_20190907285613_000795_zg_c05_o_q4_scimrefdiffimg.fits.fzTESSͧ׏t& qC{tD}ao@dQ50J@^A1>?rAKw>>Aj<?T<V`1 >1@#y?qCԝ>?IL?'?8>FDq>?Ew?t?BA:#>`Co@L/1J@-$@9A\7]=Ʒ7t17_f5_c3M6BAק9ҝA/ztf/archive/sci/2019/0907/286389/ztf_20190907286389_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESS< rA67r> =Ur%7t17_f5_c3#Qa7BAć9A/ztf/archive/sci/2019/0908/263345/ztf_20190908263345_000795_zr_c05_o_q4_scimrefdiffimg.fits.fzTESS̚t& CrDLo@+/J@ΙAO >?AZ>bq>!A<ˡ?=RI9=ף @x?C +>N?ffF?ˡ%?r>L>R?g|?&x?VA>7?A_6>5 ?nAt=ff?/=#2@y?owDZhC?]?#y?d;_? ?>r>=D?"rn??Asג>,o@dy1J@hoA%@A)6Rd<^=?7t17_f5_c3\d7BAڇ9}A/ztf/archive/sci/2019/0908/286852/ztf_20190908286852_000795_zg_c05_o_q4_scimrefdiffimg.fits.fzTESS֚t& C=DKkfo@#0J@՘AP>'?1A#9>=Aj<?T<.K^ +3@Hz?Cё,?? ?Qx? *>>?j??A?5>`Co@L/1J@ui#@mA\7=Ή*:7t17_f5_c3}d7BAʖۇ92A/ztf/archive/sci/2019/0908/287639/ztf_20190908287639_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESSوךt< y!sDDЂo@l.J@A=(H>{f?AA[>o?DA㥛<G?)\=Tc9zd@y?y!sD=>W? p?nR?>.k>8?ps??A>l@o@daU1J@\m~$@A9{7&=M +=P8t17_f5_c3l 7BA9QA/ztf/archive/sci/2019/0909/262025/ztf_20190909262025_000795_zr_c05_o_q4_scimrefdiffimg.fits.fzTESS꺒t& =JC{D o@⢤.J@QA-_>X?7A!>Y>!A<ˡ?=#=?Ev?=JC>dX?}?5?l'?˿>+>?jy??+A>?gUA>#/>nAt=ff?/=>r>W@;o?LvDz5>I?XY?+'?>akF>Qk? +Lt??гA;>,o@dy1J@bbQ!@bAo;6@P<=&@7t17_f5_c3V7BA̓9xA/ztf/archive/sci/2019/0909/285440/ztf_20190909285440_000795_zg_c05_o_q4_scimrefdiffimg.fits.fzTESS񌝞t& qMC)\DE@o@*.3J@YA>S??3A>}>Aj<?T<IǾ m>p-@x?qMC6+>-!?-=?$?G>r>,?Su?p?A]ܦ>`Co@L/1J@7[ !%@dAs75=>Y1?:8t17_f5_c3 +ף7BA9״A/ztf/archive/sci/2019/0909/286215/ztf_20190909286215_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESS陞򨝞t < ysD\DCo@L1J@At'(>oM?A؁>L>DA㥛<G?)\=>>= +@Xy?ysD|~K͒?SC?Q8?<>4>複?x??#ʜA?l@o@daU1J@~7"@ Ahp79=;1~7t17_f5_c3_b8BAѿ98A/ztf/archive/sci/2019/0910/271725/ztf_20190910271725_000795_zr_c05_o_q4_scimrefdiffimg.fits.fzTESS᳍ݡt& yC +Dcuwo@)!XU/J@濚A\=>b۪?ݵAS>>!A<ˡ?= +==@@d;?yCB$=^? p?ffF?>9D>?Qx??A_>y?aCAv>|>nAt=ff?/=^?}0p-@bx?ZwD>г?33S?7?ݛ>ߦ>? +Hz??DzA&>,o@dy1J@!'>!@)mAױ7 += =r{7t17_f5_c3e8BAՖ9XA/ztf/archive/sci/2019/0910/295035/ztf_20190910295035_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESSʙ̿t< `=tDDȠno@M%M/J@YA3Q>U?BA;?2>>DA㥛<G?)\=Y>*=?33@v?`=tD Y?6?M"?JF>B">? 6U?g?v`A?l@o@daU1J@n!@ADǍ7=6.8t17_f5_c3Z8BAȚ9A/ztf/archive/sci/2019/0911/286354/ztf_20190911286354_000796_zr_c08_o_q3_scimrefdiffimg.fits.fzTESSt< 7ywDqDo@)!XU/J@AA>E?_A>.?nAt=ff?/=f܌U/>@= +w?7ywD(} -?rh?nR?Q>>>0d?X??4A#>,o@dy1J@гY@^:A K7M=3=N7t17_f5_c3QKs8BA㛞9A/ztf/archive/sci/2019/0911/307928/ztf_20190911307928_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESSԩ津t$< lrDqD繕o@M.J@cA>"a?e*AV>>DA㥛<G?)\=2=Eξ\"@&1?lrD#d??Z?q?l@o@daU1J@uzN?xzA6~>QO=Aj<?T<1>Sd>@Zd{?)CP><ڐ?;O??5>?9>>*?jy?3|?:A\>`Co@L/1J@*:&@A27?=fCش7t17_f5_c3cS9BA95^A/ztf/archive/sci/2019/0912/153148/ztf_20190912153148_000796_zr_c08_o_q3_scimrefdiffimg.fits.fzTESSt"< ~DHDd}o@/J@ `A3m>i'@h"A7>>nAt=ff?/=/d>j5@!r?~DSyg?mV?L7I?9?>W>g'?o??Au`>,o@dy1J@ͪ&@A6<=7t17_f5_c3@W9BAΔܤ9jA/ztf/archive/sci/2019/0912/181609/ztf_20190912181609_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESSɴ€¨t < {T{D +DĆo@BOx.J@ Ao)W>9?A>>3>DA㥛<G?)\=կ>8>H@/}?{T{DKS N?m?= +W? h>NR>ŏ?Ga??A33#?l@o@daU1J@-R\5!@A{d76=(={I&O8t17_f5_c3s9BA9 A/ztf/archive/sci/2019/0913/151910/ztf_20190913151910_000796_zr_c08_o_q3_scimrefdiffimg.fits.fzTESSȩԴt< l~D{4D\o@Y/+/J@5A qe>x@ёA$>f>nAt=ff?/= >n.>?j4?l~DwM>@F?!>P>}\>}Л?2\?q8?BϚAS>,o@dy1J@R$@gAo6'<mj=(7t17_f5_c3/9BA䋚90*A/ztf/archive/sci/2019/0913/185428/ztf_20190913185428_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESSҩڏľt"< }DDtgo@zi/J@IA%>j?">DA㥛<G?)\= F?˾>z@z?}DI;?EV?VM?>Ű>?Lt??"lAM ?l@o@daU1J@[_$L#@A 7F+=.?'L8t17_f5_c3xS:BA9 A/ztf/archive/sci/2019/0914/152060/ztf_20190914152060_000796_zr_c08_o_q3_scimrefdiffimg.fits.fzTESS¯t +< |DXDo@|/J@A(>C?AФ>쇠>nAt=ff?/=V?4=ff@;?|Dؙ>.z?#? ?q8> {m>? ڐs?=e?n4AZd>,o@dy1J@ hB@VA6_<"=pB7t17_f5_c3 W:BAӳ9A/ztf/archive/sci/2019/0914/185185/ztf_20190914185185_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESSƩʜѯt< \|DDX+o@5a0J@jA)>4.?AM>>DA㥛<G?)\=hͧ!=ff&@z?\|D6>q?z4??Ԋ> t>P?6 v?[?KțA ?l@o@daU1J@t"@ףA7K ≠+8t17_f5_c3LR:BAݺ9nA/ztf/archive/sci/2019/0915/135324/ztf_20190915135324_000795_zg_c05_o_q4_scimrefdiffimg.fits.fzKulkarni΍֖t& C +D*6o@ /J@7A@>uL?iA?$>dp>Aj<?T<x>H +@Hz?CQv??L?^)?>nn>.s? +( y?q?^KAY>`Co@L/1J@d]Kȧ)@XAb(7) =OJ`W7t17_f5_c32y:BA9KٟA/ztf/archive/sci/2019/0915/152095/ztf_20190915152095_000796_zr_c08_o_q3_scimrefdiffimg.fits.fzTESSƻ鈳t,< %DHD=So@ .J@AI'>{-@+Aྎ>ּ>nAt=ff?/=~ڽq>Q?|?%D!?`? ?j>H>I>b֏?_?Abk?ӝA>,o@dy1J@(\%@A6'<=Sw07t17_f5_c3P:BA9A/ztf/archive/sci/2019/0915/182072/ztf_20190915182072_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESSҩ讣t"< |DÝDu1o@)Ojy/J@!Ax|;>!?qA0Y>lj>DA㥛<G?)\=aY>!(?y?|DD.?㪶?j?E?>>;? Me?i?;ߖA>l@o@daU1J@!@A77/=g@7t17_f5_c30wwS;BAɡ9ہA/ztf/archive/sci/2019/0916/152025/ztf_20190916152025_000796_zr_c08_o_q3_scimrefdiffimg.fits.fzTESS< nA;ج:=(t<N;t17_f5_c3R;BA9tA/ztf/archive/sci/2019/0917/153669/ztf_20190917153669_000796_zr_c08_o_q3_scimrefdiffimg.fits.fzTESSt< }D`D3*o@ʎ.J@7Al>?KAcZ>>nAt=ff?/=>=y?n?}DFQ?@??u?F??^-o??Ax>,o@dy1J@`< @-CAS6<Y=fr6t17_f5_c3כb;BA9;A/ztf/archive/sci/2019/0917/182639/ztf_20190917182639_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESS㚤t< y|D +DHEo@F1/J@6A?>?,eAoD>$H>DA㥛<G?)\=Eнz$@"{?y|DC?|?@?.?Е>އ>O#? x?U? +WAF>l@o@daU1J@/'K'@KA?C7=r7t17_f5_c3S-Qo@/J@eA4/>f@}A:=3&>Aj<?T<>T]@w?BCӽ??I?V-?=>n>e?n??L&AW +>`Co@L/1J@W[2@A^6<jy0%7t17_f5_c3櫉S6?vAH:>>nAt=ff?/=4?;m{?y?׃|DvZ?)?Nb0? +?>3>:?Hz?CXy?Ad>,o@dy1J@D$]#@zA[>6dr<=7t17_f5_c3AWҫ>DA㥛<G?)\=g>{>ff@5^z?{Dl?? +?'1(?߉>S$>$? w?l@o@daU1J@/'K&@WAE 7=3]m7t17_f5_c3Ί|@A>r>nAt=ff?/=G>&8u<@z?={DP??+??+>> +?]?8G?SA2>,o@dy1J@5l-!@A6I< =v7t17_f5_c3aEû0@A$=T>DA㥛<G?)\=t\mH}z@m{?{Dl?!?9?/?R>.>?x??wAvO>l@o@daU1J@Ǻ1@YAD6e<BxԽ?7t17_f5_c39S=BA⦎9JA/ztf/archive/sci/2019/0920/152373/ztf_20190920152373_000796_zr_c08_o_q3_scimrefdiffimg.fits.fzTESSҩt"< syD^D+?o@5.J@w>A_>?A&1>?nAt=ff?/=]n>)\@d;?syDj?w~?V?迗>?w??mAT>,o@dy1J@io(@A6o<=&[6t17_f5_c3gZ=BA9GA/ztf/archive/sci/2019/0920/206227/ztf_20190920206227_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESSƹ񺗃t< #CxDÍDe+yo@ G/J@AA3=ɫ?A> + ?DA㥛<G?)\=kBw={~@ux?#CxDf?-H?+v?'1h?dx>4i>?o?:?ߘA%>l@o@daU1J@ h"l8-@ΙA*7"='ˌ7t17_f5_c3=BAֻ9%A/ztf/archive/sci/2019/0921/139549/ztf_20190921139549_000795_zg_c05_o_q4_scimrefdiffimg.fits.fzKulkarniɫt$& hCfvD*o@$b.J@nA)=!@ؘA =>Aj<?T<t>ܕQx@'1h?hCĴ?+5?&?R?%>wY>?Z^??A&>`Co@L/1J@Bfj1@PA-6vV<X6!;7t17_f5_c3_K~=BA9A/ztf/archive/sci/2019/0921/152234/ztf_20190921152234_000796_zr_c08_o_q3_scimrefdiffimg.fits.fzTESS¹t< PzD`DlIo@Q/J@X(A>h?Aq=J>y>nAt=ff?/=Uj6=z= @Xy?PzDP?^֔>? +Hz?u?AF%u>,o@dy1J@V&R_#@kA!6^z<-=7t17_f5_c3A=BA9濢A/ztf/archive/sci/2019/0921/181644/ztf_20190921181644_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESSƉ߾t< 3[yDD߹o@Ɋ1J@A>?A`>x)>DA㥛<G?)\= =n=H?R~?3[yDu??(?A ?h>ӆ>?jy?0e|?tAoE>l@o@daU1J@F_X*@GAd +77=8ӹH g7t17_f5_c3Z]S>BAǀ9w>A/ztf/archive/sci/2019/0922/151238/ztf_20190922151238_000796_zr_c08_o_q3_scimrefdiffimg.fits.fzTESSt"< RzDfD)ko@`/J@oAh>)?;A=,>}>nAt=ff?/=D>?w?RzD,?!?ף0?G!?2v>>0? +du?[y?A0>,o@dy1J@MJ$@*A6ו<1&=7t17_f5_c33+%W>BAۜ9ѝA/ztf/archive/sci/2019/0922/180752/ztf_20190922180752_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESS쉆坌t<< myDD~o@0r.J@H?A{ >?VAz,>`=>DA㥛<G?)\=c:<e@-R?myDj/?_?x?9h?>>&߈?Զm??EXA>l@o@daU1J@H.+@Av77=hA$8t17_f5_c3>BA篻95oA/ztf/archive/sci/2019/0923/152442/ztf_20190923152442_000796_zr_c08_o_q3_scimrefdiffimg.fits.fzTESSt< {DDԆo@G ^/J@Aޯ>?aAzl>l>nAt=ff?/=q>le>Q?~?{Dҫ!?ŵ?MB?B`%?>$>c? + &z??^A0*>,o@dy1J@x@ @mgAM6*<4=s6t17_f5_c3V>BA9A/ztf/archive/sci/2019/0923/182280/ztf_20190923182280_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESSt.< 3zDD-o@00J@~A\>]m@AԚ&>!>DA㥛<G?)\=7> + > @Zd{?3zDh??D,?S#?ۜ>z>9? du?hr?yXA]K>l@o@daU1J@+@>hAq.6< +#S7t17_f5_c3{S?BA9@A/ztf/archive/sci/2019/0924/154109/ztf_20190924154109_000796_zr_c08_o_q3_scimrefdiffimg.fits.fzTESSt(< }D=2Dᔹo@)!XU/J@"}Ai2>?SA{/>>nAt=ff?/=t=Й( @33s?}D ]?\?D?r(?>ՙ>e?J}??A>,o@dy1J@[B>%@Aa֥6!<7=Z 7t17_f5_c3#`W?BA9iA/ztf/archive/sci/2019/0924/182581/ztf_20190924182581_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESSٯt< yDhDs޵o@$.J@=,AXq*>T?әAgD)>%?DA㥛<G?)\=>-B1@`p?yD?)Z??x?~>@/>^?\??fAj>l@o@daU1J@ +&@!A,7H/=zp8t17_f5_c36?BA:"}A/ztf/archive/sci/2019/0925/139144/ztf_20190925139144_000795_zr_c05_o_q4_scimrefdiffimg.fits.fzKulkarnit& orCDo@߉Y/J@A;=z?nA?>휶>!A<ˡ?=X>'"(@Hz?orC6;?@?v^?d;??Y>n>? &z??mEA>٬(@QkA6<=7t17_f5_c3$n?BA:A/ztf/archive/sci/2019/0925/151759/ztf_20190925151759_000796_zr_c08_o_q3_scimrefdiffimg.fits.fzTESSʹΒt< q|DDwkŹo@qH/J@ྚA_a>?,eA9EG>h>nAt=ff?/=τ>؁3G?d;?q|D8V?ު?y<>y#?( y??WlAc>,o@dy1J@U+"@oA6Jƻ<=-6t17_f5_c3W?BAśɅ:A/ztf/archive/sci/2019/0925/180220/ztf_20190925180220_000796_zg_c08_o_q3_scimrefdiffimg.fits.fzTESSʙᙋt< UzDDѹo@][.J@A>n@7Au >!>DA㥛<G?)\=m= >@@R~?UzD~`?&q?Y?G?>^J>+?jy??A44>l@o@daU1J@jMSt1@ cA"7 =[1(}7t17_f5_c3vcandid1000151433015015013_pid1000151433015_targ_sci.fits.gz׎]UwLKuy=8wwH` pVrγog_o }LV*mEl?*Yfu+K?Jkϳ7?/畬U|7RyR^{?Y{4^ݦLz5ruTv|-ƔDgyN"Oط8}NAc0!o=b1]^nJÅ;o_^R9#?|Vf&:⟸@ Vh &bzfq\Y/5,P"E6xN˘x0CmV?)5-yD$Z"a'OglBG݄ޖ7d GQ&scI*Ϡ^j/\5000c0/ēVxvjTkfA|Чe>E:vi#fWp\CrZW0s.$/#>Bc4U(t7Hg?SoyNjv!޳3.ȳioq́(bpS,̱Kp%'ʏ(计R@0, x*]- +kG49,&0*S)N3%);qՉ9@#4;NEc#4)-(_DαFhce}I@dQ'5>Ɍw +b +^Fe4IF}!gxzuy؉`QBb^LBT`,يo $-YA/Zo3".>n(P"ol3skj"%~&ЗB` 9zPgPX=c\h>j`[1`\ È\Z2cZX5N[${zB{|i=ʏ!5z 0Gtq1F3&Z1(]OT9]mN2mapKuv|mV5>ԕolZɗSPJKN(Ckphƣ|Ȥ(I]axʹ=8fiĪX_.Ǥ_ {y.wa~+wԩସ8bc>Y7p#zb.ُ(&Hn}CdLk`䫎 ;|Ђ8LtW\^3< uӐ24ߧ5 y^êaܫ3bG`^ލrssƍoT7B ;_H,]V5ȧn'nY\ۦ5֢8K /5D >|L,-W0>.;dyrdx/D+\ejF!"}4{F5jfΡd'ǬQ҇N';򀜙dؑ׶amv e0zco. opWF9G7NН1}F%O.ɽ L̙ٗۗGH,uT{?&JN{<%y5üv-jTy+6qg>r!\wп;{rˈd\ ah4 щ,5B 7?'ݑ4Q[W;=s Hc*_"0W/o^P:$e5/7JjYQkJL8o]g';,jصױlb871w̙tem*cA+zr6 pX3ްD4%16JX{6`ua"Zaay~DPU|퐯h%k<9b]o!]P'?w#d̲5oG;)$nw +w/J1&=H̻.d^);Dj? |;'Y,@mI=ޅ sa%p"nɼJ2%u3i> qVA? +Gk $.[W>I4hI Bm SFD"t*v2ctq뺣~.?zb{%k`&M+F"ٵI0. 3 w; a4ź*5c +|qX[KEy\&ʦ'sٷst*:0nuތ9a\kd'Vo.$yy3KPrD0ShT`;ɴL.{l ڛ{.~nȟ|ug G'dOXM|xG+-P7EAT<}0; ,Β9ހ+0|XṘP|حM 2O;9L}DW-nS|º;i3^*.Qzv](a'ѓnP[#0CD\J}]SޠJo> +ĴYB- N)( #xax[Idɮ1WȹO[|2Qd_ +;!LY(%vDSAS>}+ '&&0;Mk\}rP%oHkEoUZI5dVjDۮ#8Iz#Y7Q#9!E` o{3{?3o,gm{w*w5<[a~o?;_toE-*ʣ.( ZXt񱸁NlD_+0> +w?BO-gw:Z.^~%q*Epgg9gndhD5Mt5ddxRjNqGF#%/W1j6^g`-!;=̗ gKr Bo]mӑwa\#yպEXMb~N"Ga q}E`eφY/b[!cmIG`}ܫ-݃K5?/^v'M\wK͏JS^&جpJIږف-t&xkǛ$L),:Og9,N2.4ޅ( Ѥ,9oe3-,}[;p aTψkCdwvOh+?]gS'➾Iec3`O91 l/L8=Jro)JT~;WvQ&9V>_>yf7E4 +D?&A>{Zf8AU.Jyq7IN_P3Y2d3ٻd;&ܠ3QGcsտy~ߓ/`l*n<&8T7MO {wѴƘWY&;Dzfk$o+y¾&>$ѯbYLX:g'oAamQv.vg-^AX59ƂC(_b~;' BGв$EY]Y2zޑQwLLw +koCu3'W}$wfFΐ0-΄Ww+J?:S ȱcy6wob $\d8p{ڤBI'%<`J"gdGI g,IGt6ޖ~τzq/u v/$F8(/V#d`.Jd`B#q+5t#V<2z y%zLLn}gkMbʎN){Ugd%&26B@Ux{m"gK)[δlS+1[ A+q%ye'>&4P#8f/a񥩸Vǟ>22!q1lSE$b$ȵ5u.~_r%QjId۹ySH?jUMrsSx_,CۜU:ΈŹEC{~;`?&qHZYw|;X<<%v';R$J;O(/ >jOBM,܂1x]bdEWYedUaˮQ=7aPG uT +ji񚢹Q?H^|?'tJZ(O2;= +VsM4Ȯ[`ŶźuT;z!">iX}vVAOPv_A¹D|B2>S![+7ڤ'MRSj&eNuW%j_IIÿ3昕+[}&tpXq1NFܱbô?c͚t-9[@KRkH}4CA|ۣ1WS=&%AK6k{ |cU@?ų.Z.a ;~#ہ섮 TlAemՂpbh9TA7>0!*D\ W9u|xV@E`ORk`x]y Cwc1M¬\ Wh7˳BM]aK7S ~uLC˽Zد +>?/|{۾od1"9]{W~ ZCU|/gtUʋtCɞUP)ua\7g"P\%pw Y{(=?mAݹ3J'_`/]q&j/(o{};ʺZ(Ĕ]qyӱ?kAe%u<4b⃳?58 mHk䍰&}IđYϖ]dG;:zF#VCw6K?l }9 W#D/!=ywň,b41Yl\3_ktaCcMI9+'A~ay÷D b!Sґ3==J¶*YDRĄUƟS;|YTRgeꓶZ%Ը8kSh-#K=< <f~5x{k5ɮ&hkaem(k?Lm,u,ce=qMEx|G7 +K +2 _ zA13g["z+Ok8k2AxR»8v #kQ +ᕅq?z^5jA\ћG!akbFYd]Sw.ɐÉU^,#JhqWn|.`kX户v\> +8%J󑸳TAxqS\% ]G݅d0üO:w;ќ<7NxpSR=ז(DŽ}붗y*N +N, b[ypD ;BEhX5ˡ_'] ݋IQSW}؟2bc g3ʍ3h2=0Lɋ~^% }A\q5Sbfjep<)~8otg^X±1 DY; e`JɅ r@q6\W1ygK;I0?4U:p.O@>-~ ]5 3HC5lԷѯLYRa |E}@7܁7FmaSEJfaR'_`zt&A',7oꣴ 8$[ss7iHBo(J$?D*u"ۉ^ɽ+A}jl/% '.3>ZφZS ͊Mcp?'GvQ[}QcZL0H*8>Seoqđ[a-ϙ؊^e@/֢=߽B{ ̹SD (A`8ҏړж8%c 抙(Ya +7^Kg\|ҤCq }zN,o9-'mdϚ|ٖyr?>cPafrMM>)#CģG$g_-O)@}sb}vXOsG +}2z;W#03dRMeW`ގ?Y$7',9_Ů`u$نRֱU=Gѷ~ϯQ~Yotfbguce&.b.R!*p!,]-'o92gva zKqU/$mLsUQs]@KQ>Ù$ݞx ?~|>3p,w1OүGqb;|!&Cc폷y#q'8cs-/RtF2/mfjxx"aekv|~'Xf:k<_];ziroXf<I"r q5߳Qҏ?ΰ(_yη +ϴ&mEVct,?9ZZ0R,ڙp +sFOfy4wbz^©R$%KUݱW$Iv4r^GZuU](7[,[r~Pc Η<`$F 1!xf BUɽߠr?vvoy#C(}ʳoAXzȎCO SB2'~kuK'0w]ғտx~FbIO+ahq2> z+ ݄s2{4R9ݴ +k{Ɯ;o%pٱwm&*3e '~"<'F F`G7i7܎'8³ KңsL*UOJ ןɄ#t"-%CВkhć_:@ 7xߌoz/DžjWաv-WPRnROL[y$t" EGKc"VCB{f]6D=)'9&q%UojmSMV%/^OGq +d}2\Ÿaw=Tѿ.Rauᩫ>_`6M^NpoRe/qˇ~f3-0@S dLEe^Y#/bǮFo{~bXg%Iǥ[1Se]Yo.(ƚIX)NpZ#3 Qv/Xy jbxiɫw߈ZQQo;V |0M邆 W?,Y)YT!|6/zz6qu"j92eIxa&VbO|y4_X=1cʈD@7}N[,ZkZecq@X/ MO>>.p:Hh||.%,;I +W&z-Xϒm =$l#֨D78wzNOˋ7,DQ@7nK&ETBJJga%p2C tx+hyo5D]ڤ/ _hGcwm[§N%Տj+zGi\ M>IK˫iR^}?NUOyF'P;H6]>ӋWeƳ29XueG|®!r+ nxjL(/{E| c[@<!R" 'VWը-B͢ [VsנEhqg޷>Q\wͪ+_>k +kE;dT欏ղfׂDW '-;aƮ_ŇdM{ Og1$}_cʊn<u)H 7{K[P=!x0ݍp,>=@lN/.]0,։L1~FQJv@/ݜ@q I$#2MB:΀d:c5T%0UX?%<=̵b<*/sjw:3rwn'A@eV᷊:+3`=~p{'b׮ n6 fF_to/Y!hq_6|?150_/9燱x=gʎd qi0⋔{~>9 +s}ĥDoadiDG So1z $zɲ8/`TأY>y0/ &[};=Cy-\B{E'(aeɆ7-6^8߻)NJcandid1000151433015015013_ref.fits.gz׎]U׿qwVUhqw4Np ٯOnGnG2Ӛ4$gV^2<)(78 d$C9woȞNWf8ye׋&AZ X Bue:2Ns9''l}IxA^3mܰ-#ky\P7@m +EH.wd.XrNNrD7GV=Rqee's0Dnrܭ{-gT<4}(sLd2m&mrkW_EVPYSdf;S2mvHnre;)s܄)ew{d,gocTVЭL +ƫKUQ|RͲt(5ʭuRӕBKOɒ%gkY9+ +y[NKrM&rܗ*XFW$ݒ4_@fuf9tU.e +q戎5=h}3e&!w[ٛ˔,"dA;)enf9^LV2.1;d ͣegSgcZPVVTO" +>r@Y\QgYT6TTBʔ.]UU~,/k]/Lcz=m"7ŗa? ){N-Y^9Ō`w׋ufd&Wˢn x|.氜Xhk 9 O .{r1y]$ Uđ#ܘ<=Yn[mGӭ/ʸ0g^|?J +$kKC&Pk}XB]ts(#.#r>֐3} {g)>?Qfd=7&E^hQK;%&7 >zwINԟhtj_-r sB꼒h`!v%zK{ {{widbWO{}feܕU%?w2ݹL^rU@?P>̦/akƾ^%:sY gp!]+L|Qѳ̠WDw؇Z0]r }\N%yz)rrQՁq<{0~SyiڭjrS#o$pe/2kU yf. i;'Rylztrms2=Lnr'OeB/~5NΤ:t܇p2FȼDhܶWiBɞ͎%E %m\ON/aqYf742)ىgLYTQ{0+.ʭUF@9w mof + +6R;f[9LSd'>M_伀=ľ>p.Fl+{2@t=< +xy\(/Թ܂䆓6,SwN.gxjzI=̐sLf7 ^ΐ7}YvAu J(Q<5L [9<>w0r]d&÷25ɒt2*Ǚ͆|O%>;#\^Yf3Z4O`H<٩ +kˊģ2)װ߽e$3-.(3>3/o@dr=]0v^Fȹowi9QԦ-<.^#lGud%Н2U굪ta_ײs׽pywYLVzNV)4>ǓA^N4!3{lL.9B9kw\g4<𴴥k ׆o}O=(C =]vrkRK&M9KɞUnAx]m9~4xs߅oVȬK-sYOG2e >o  }ٴC xLscNtg7N3wq)hGGجU^k'?951XvC(q@kC}= }a' +0d0;</Fr6dDSxyv|嵬i|:%L%Ss쿢ly,q|*̕Мh +fk^q/FJ>"ww>GW|7v,-'U๤r)ͩmVf0CAsSד +c!x~g3CJ<+9Ye"Kt}S ZpSu ~$%d;֖[c,~!{zi~d-'}9U7nTCvpk&7+<G:bl QS@TNGs`@4fN.H\Vc^qf'>Kt>[}T^~=~FљgF2Ks<; 0:Xό._*5 |AخE盡aSȺvs8zrܼeFj0S4.LͺȬ&@Wו0{^;K)Yk8hZ3 }M#mbϳd]K/?8ZN#Gz0Ǧ1g}pS^doҭ;ΖG}ć~7pQ +|4]a )[F9z>Wh +|]^j\%o^vF^8& ߉Sge]!m܇8|8M)J=x&z 5&-uzR&쿄MC-'ùʾ#-ﮤTT~Z!؝W&h$ty{X&saNu=o9>W^%cdg6'S-1W:zMy?8;r ؟|{UcVc&U锞Y-o?>nA2Co%je +lI// 2pemP<';TNp ab >:A0fJ2R@?eFi+_#eFcepP9_w0 ƯP,sؖ,z{䔜"7hv {.&#\"Ny[}rۏFa}=Vv1oɺAd43yb A+R<>e߯ٙgn,7*We2lKʘ2-r~=-F]y2΁ޟS;=DyhL + {ׄｂ ^.;.,[Þ|(+m4d'aya|d 5Ǭg= p3~<-="7'c6_C5;z3r#3KɀF^1Gs}ßwy(_v2;Bn?WZχڢZKG_.`yYS8W*5p?܊؈WEak"t/Y';(s2M&/~$LmԘ& l9_oϵJ r +܂ի<7NˍC˻Y#!KMG47S1܄Ș0vrarP8;伄wC^J=֔i8Q `Wrikxï x +x.HQ$'|j&+\.gZJ +{p-?%#^D0M|] +٩_4x9n]'22`de`sӿ5ɕr椓H2<%EvC%J?s3dBˑx }HXctg>6$uS4s;:\c+uΑ+LB㽹wfGdgLl2 rjY + /*Gѩw2hjxrІ o =*NDܕѷn`r/0'Z˙`\r"|9{\Ug> {N!ⳣm/P2Wy4쀿eCaН ؽwr#S5^t=}ޓI%3癤 >*l*.IFk/F|6̼r>%3|+!o(MOѾrx*oYzPNVd}wa^ܥL2u>3DV/y6Df7l +oxPnu&lۙ-+Zpl4s9.eZ6x!Y;%z]~BdQx +qI-)s ?s3Ѝ3sIZx~LJyJ(Lar՗:Lɝ3t{z>y?0Lѱ =lN35OCiٽF˚@;D&XOؙi'}b"ຌpThH2a,ǿFd{;SA)(m2eHNuAx~1xd_!6\f5J¿6W$de.AӰqrr^\{م䑡hr1ԕ*L-_/[$S'rӄ, g>,ܤRo,]x4M#eO$qSY5a7'e3&֑5N޷0)v=|< m\(&-3=~ NUd,`7Kby#&y{MΛPyqd< ?_E.@]y`Wx`+\u,)_L~>@גYGH3s'yl?)|3Y$zSGuѿ]km z&ߡp.PN4,XFkIx~&)< {";wSrly* |{n%7;.~D 5r[qo1ڛLx6UdhY +,<;=Eår?Iot.^hl/VαCm#%RvC;坍'xQK>ތi[!{ =z#^xJ._di ɹFVΡe `~5ed[hƀ%,‡ +pp9G/ZG𡎣e{&-.sB^ ʞ톤g`I}a;HO2TAyog=rf\f?y$'%,')sⓉIg=}Ä0uԏTYc.;iMuh5 ^xq7L$8drP ^|+ƣ?o}|2ExY͘Dz.ѓdȯx༜7eOo!'R~ ,YK3k +*H6?OMYWiJRa% WO#{;$}dQITj{=' Y׶aO"Y`xG؁ޅs' >W[Kn°CF3̈́˿0ydHW9NaAz}N(9x㧗?8`Ͽv^EXu7aRٷ{Áء39+sʮuT̲&y&Κ~@g3qz5ֹB}'- /qLl9g.!239c|?``L9Xx'.,);=*NS˲kVegx):%7|*7:vuXv[EO_-{ ȝVyJ:^<YтUw"nyE6;ږ",.We=>%z/ Jو=E٧q<{ۏsn41;tcy񾜝o&׿U8{x2ٮ%jS~8 (7K SlbpwdeOjLY"%;my?9&w{ɛZV%rJ,򒝕.Y+D۳Lx)2Ģ2de!VJ)c9sH}8 +?ñȸd:t›)d>'{۠? (X;|S.%K\MBt35i܋q0P& !k ΰ4{:s ƕȂ.a-rƌG.C~\&5 /;GYYKreoG{KʉHgחo:0̼2uعd=bۭ=g^psm?ۘ[p堗#+cOKݐSw%(9}&cG똃aeOKÃK +}'-ZVY~QGanhyثvdO':Y{qe{YNn2kS4ڵN}WU1!^<E#дgH{(ߍg4hjOrџiVOv%ks +_IN=$+o 3eΗsv +uVs1d'U2{e7%)7̌ ̲(T֎b=~ҿ@+@1Џ;H"7b9Tdz;X +x4jؘYCO_FW* _\Jaڵ|й[M8}{S;+]ݫirKe,з-^_~ğmE提\V"C=NG>{2ol%{WX| ;|v uv&!۷$ֵ匁3-'ө@YxiTnİQr`0'ل vқh}?Zf `$B#F1țero-dM1Tv:-tЪ/它#ܐ_hbw!]8^)[W|{rd!$D?GeFkit.-wFᩣ!]X91|辬(:x~Q6@M|'?Brkck6ڍGwڸXv^w9J{kЯHux3g]]V.Uvlfcೌo슿NWd F2ʫFj%g*_=+DzA-ޕ ˠn]6m8 rI}nqO巤䍉xtk%[ +鏦r-3oi;1o}0b<3t1`m2E_vl__Z|^J~O-O?>#DEB &2i>ے-GχA|H tz?oU?{;TA֞[Թ8Y%]AtٻWxQ!r򌙬.i=5ۍcmr4>qm r5g|`^`OX*;^syZOn4xs#9s`֢PPIiiҖ]2xK;UPRBݻ;=[8xdknZm׮u_zO s`mTzz{L9خ_j/q?iKT????jLbaZ.i4l~vϯ!Lʼ$A>PTa?dc SNY`~ +䇚qJS{=~' rpVzE:y8G|9,u7.أE^"+ne Π*=Yzp"ڬ!hx8|aϕjU+k/p’Ȣ]9_&dk@ޟ0BVAyh + `];ZX3;kAGT vg lzc`tZ\^Z#`ԽvHJ<ƙrb|׶N6Am(FOȃ_g3ID7XUR3H*R8ŞZh9\0`):{sقoO-_2>7b$˜dMN̜yLNdo+ 0rY4W# }`bf\CǃKxVtl!j݋trUVwmإo/-aӖXee;9rYsg+X**t 3^6'Uaj!VW#5Fs 1{+?K+%9;C֌o<?X WT!~421߱׃]23,`;AZ$ꋦd-i8Lˊ̿l?'\wp_u.T6C$z*Y b>'E":c@"aa}h~ +7x}¨Q"}oN~Ѓ ;QZ#n900P^hZ(Dt!ߴ ӟ\:Pd2>M^|WnjqClC{bPj̏%?|JgIPimUrw-B֡lߥU(1;W Z J8JwRb[vk-Q"@4fJE2uH9`E䒞yܜ.EFNE ^lIJ:FRyjyw6_vi& k=ش\x*q)ƒR`cm=T\NcY ,<94='[fL.%3ϳN kdM 70 dtaJ*k[;"c >)+i[DVr%Eȟ{,vw5ƒqM3 /ƎT}zސ!Rw׉tC՚*(u y@A7Lئϊ'BTU +J7c켭T͆n[BߊD01r9 `&'F&!E5|J+t܀Tb_n&;yF;; ^[["2oE-?nv#m +,QYj7}*#TP7 {n.Yg ;뻠 Ad 6SQ(WB d.&/SF_T ¨IJm5sQ%+Q6 #/C0m5Db6hsϕ]gyո*OP%}#FNK]ZZCFj ҕ5 +sAeϲ,JܞvZҍjJdēu ˬNCrf>#qn8 +3/O 4 WAћ6CFhFP~ ǭ9OG/A+~a+PQ?qpWgG`_pᾛ짊q!˿j*ס+paY2aCn6G !t:Tj'P]3R Ũf.l1񠫴kMfm.k3S?ˆm|$B?==`;\xȥzݸ>؉_gj- vQ +nd5c +(CrXR]CZF<@S;.VvpjGFoxb!3; +%g ]F=Q`ڼC{$v6*epP.MM%QAC|eBnN8k=4.׫ġ˯^ +E{xEK,~P՜ (s2s 18*pX܁lSj6dUkZFuYX., u[nk\D̅)'zD^0u͍G8/߄h6d/pofbVl\r)H:t2$=8ܹ oR:/Y&di' ڞF'g:QH |ؠEFƒ,xƇvs>k!EڰM<0´H #r(<Idi'ĶqDcpq+_(w,:g8*ܼS; +0)3yY]9 h V8A0\- <'"VE(x|lS#̊mMGCtc,+Z ݸ* 4#ˡvl8^O["0ŒQgf\QST1ur+~1V˺:``! !8V/t`YZʸu{l+o8b.rEvN,/RP|J2ae,Eܴ' Km}4mLͳ+Tg)hwtVgYCъ߁'Xʐ~0r J EW*ANbR H=cFtSq/Ǭ I{^6Δ$a+dd@ؕҊ#OB3Р!lо.i"d j<^u)H^|o m*C؛JϾNTbeL<~奥0fcx>$rRxTLu:(6NTrЩc/ᡏ}s\ȃxy@5 H9eM +j b{4F&du,8@/ U0mLR6}<)рYh.k*h^3Z>^a{1?l>[=hg#g!=իt3&VE̬Σ4ܸE/UǨO5F1zNfV|mnyr7;f]WU$KamfkOf{!` +nre;8,چ +` aMN>ëu-Zxs;N4P%F|e#?.s z# Li  +|u?] +r t7fɟsM\̪ + h>6wQӠ"fto p!M?~@ΑO"RLFsN$D+Y. N̈́$d'l/p 4 mc/ip4WtxtA )II_ZW]hB'By_њzJb34"Aeb7_S[ע~u5=g,М!s>v(Xif6#^? T4v +2{wѵJ8"p) +3|.҅# cV:<-]c sV7㪌ɒ֐v*xP o9^y#;WFSCRz/& ]$3ݎG*#a}Xq, dS9^&F =(| AMr[#gEЍT_m6,=0:w #uj5)XL<֌_VC3r_bL<6EAO[I8܏_O`( AFVqH3|S, dW_N{5߉C \W@R,!ZG@ui+WVl[l6#?/mÔ 3NANf'P:83n/Wț%u(<ܖ6w+Ӽ@X8)@\{DWnEzNC92"w&hX[mXv/k;{rzΊ/hٞ*niDՃ0PQ"2g=6^_qb4ǟ3[1?d>m6*sKptF?T8 ЕsM^#d (7VuabN?qPE"p7cHK*gJCo=Jo犔!F(4AI#< i1PZ9L?}:z}4,-cp1Ρ./+C)Uc)|V9卢A#8n:&;Ie~NU=?LķY3*w+iW >8<^{{Erڬnr]Ody+D\2{) 㓺pޜ'jB +} m$prs ad\9 Ⱥ'^@'MMht} Sٷu'$&60zñHiÅv~v#Ɛ+0=Av/h|_⃢n2B-1jl4KnL.v{KHڮgI<ȻQ lcl;WWX24yUl{>a4"M? yWBK-\n@jϥ# .E oC_fh\a O3̳0zRkeg>ll$z~@4z6 {% ⺏yԿǧ߆5z8op>1y|xq\^5!8N3}=U jq?Ԙ=wT'NT%K5L#uKKE,nrԗ׋Gփ ^{|_& +<2""2P\x[&o<6 @/N}H6߉M JMl]8Kc=~4Q[B\Q9 E/\@%6c% __{ЁHxɀ +^BZ2Vf.*kVM)e`vZp?#}zc)w1;2 6 "+t)pm}] Tq~r𘍅CI^Viގja[((̎t4s9}7+eQܧ`/ke~8և>TlYd o%#aJhZyThIǺ@*aЛc _PvxqEea9 +vmd_ +5!ScȳixzBw-1Tf3?hQJlrCKKFGc-;=$eRX闀駝h[G4w$}pz9FFz3YZN I4Ƅ9o*L=[&RSU>MHC~:)' +/CFmeH<pU+Yi\\jJ#< !Ve;Yd.(Cۑxտ:Y5ƳywBwmVc+{j %Z;,lEẑ(?M7ݴՌ*H r˛Ëo4r+p@@L5 V鰪g2s7,;jewcX%V|\sag,usXXCG61 ޓEA:d,ck/#xq p2 c٥ Io;Vu5n\6yW}>^Yypji4lMOavր0~cTSZإ屭Z.L.`i#E_oOOEhwN332&Kc[9Ÿ4 =Y֤"h B]/C49kmb4}}P#JlcD-) [Kǎg\VE!&:ƐOKw5,0Whny'H1p5pՄ̳KɛS˗S{, +DUrǸj+ +MwLaD#n႖0'3OV;Ox6\$fc^\Y J Ӡq!i:3xu Wə^`,dG/otvϔEB/pDx2?<ʯ-{*FIqGzUU;2' 0 dX=nt0zV? qoPz",Ob[g2f> NK$X9mLRU X.J4uP>HyQAx9/,GrsNBtw6; ;&Z|󒐒>[ց(62^] qP ݜ"ͯ z6H^ X 72(`4~d61oAyv~'x2LøӋ$ހXsP#$/ýt @܌Y.͓`X +_b0g{ߎ‡߰6mg_݅C΋X}??6/Q${0fJhZ&J&VEpu~i_R8gCUy)c̢x$ 3 [aW (|o+0n +/|Ks]@Bnʷr 0u\ŗ釲Ca# +r7hY$B>"0 #qb7 EB5WpGDuh8 +B t+<\HظF>]7􍧂oct z~(ktNp.p 6AegͿ_nތ0jDS{}VXSN ,ś)!~9T!yϱ|۫oHJ(j1= +{$푀JU!j-y7C^%{='5,0!D]݉+taz{7̂d9 gvUVu.ckx4#fԿϟO^p.BQ J[­ׂtyM;6m nPPȚNik.i! +IZIjejwG+DRK/i|f^h&#",mO9cI)|^K5|@ܒ~HﯷW=p|3Dm?As922VciM00^Vu Gya:L#ԚMt8/-c < Zn"oINxy|O?gw+jLTsD>"W~ǟڣ`,zD;)[KDJa+5A΍*)kh.[\#ix)󆟒U레r$ #.ExaPa;Lv #i?FhUJX#A?K̒e~Q:+7'<ݷٙh*x38bQ0_ kb>2ÔSwCk|?MW|f&#ɾGAvJf03EV3v31 c z\iM`~vb6ấ|pOɅ?F![ +3:F +7;JEz!Waq&e&feuAP-$iÀB(=zjIUjy|`N`qW2g4V˼x`b&E,2ÄnWu{~zDkHl+t=,ys0z=:ϥWdn#Tƃq-z u +֖8yṄ7T:}|wrx@ o#SVBMD0a"x8@G{ R 00 [ zXQeVn9θ܇PX_L:q@"#('̡VT/9o{֯ +~B 5(k}WZ섔<̩Aׁ.sΆ>\277dG!$4e⇏pItP=G-:+\Pmv|$7(IJ!9xGtvRM~9化&$wB6,h8 +:֞nk&0{p8y0.?DX^( ++hSm;'0Y!Eߌ!#pw++'v?ōNh{)q 8#҇Ki@6KYB4K@5']37(/1^<] +Kǀ[-a8 H -ןG.Kvkq>pE1r#nryVPwn%^~`, 5ܔtpf$߳ո/}OT/6Yrv *pc0~*1cZV\9lFj/kcҊpLq6_%%\%qzU`+atp3 e=QEpz͉ +4Bsg4A#T*k js.S~%%*  qYH}=[@_J&@Ԕly+Pa19fUq@w-]υvzs/}U!S1}bfu-ܮc+^U~uUAZ8)/!-+ X;oa[B.['CbE /{LQ儏' ^}.E,n?oJb0-Q c 881ݎGр1>r"<(3َI5ݢvFu6 +w,.I~\|6 o `6d>;} yߙfO -IMx3$ő7y wс7z(XW!Vߕ"{@v"hxp@,QB py bCGMNΌy[6!kE;.FS8=k!hBRf>LVF;'][Wh˓nПZϢr3˵^Xk1B廹}?bĪf >KB+qoZ+бXˍ덇qdƅI*ד|ϗ{RL^~;:{[c IHάO%zb{cL +^P=B`tj^ad$NM`>;ܧW}`+D9R涭77 㸬;k&"%aEV4_'H^|r@^]^4¾G8y`WkȥVp/g5usX8.qBkG#WRi~u:\m ˱+CjDh,Kzy[yMX:ӉyB{ iaW4&r + I7H_ܤ\sZ.xZ~fR'5NT׬Ô.o"f[|7< (~UzzXRVI6P)-#  <+5 33p_9R5Ԥˆ=1֑6DՁ1Vd'P*jt[Uִ6NgTsV%xX.VZ+Cin=5s`gR4BT9IwŎk1{NoL[ҵ?q-Ϋ70zdPbYFN挈;Р{ \ No newline at end of file diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml new file mode 100644 index 0000000..6d040b1 --- /dev/null +++ b/tests/integration/docker-compose.yml @@ -0,0 +1,35 @@ +version: "3" +services: + zookeeper: + image: confluentinc/cp-zookeeper:5.5.1 + hostname: zookeeper + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + + kafka: + image: confluentinc/cp-kafka:5.5.1 + hostname: kafka + depends_on: + - zookeeper + environment: + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:9094 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1 + - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_JMX_PORT=9101 + ports: + - 9094:9094 + + postgres: + image: postgres + environment: + - POSTGRES_USER=postgres + - POSTGRES_DB=postgres + - POSTGRES_PASSWORD=postgres + ports: + - 5432:5432 diff --git a/tests/integration/test_kafka.py b/tests/integration/test_kafka.py new file mode 100644 index 0000000..6b2343b --- /dev/null +++ b/tests/integration/test_kafka.py @@ -0,0 +1,2 @@ +def test_step(kafka_service, psql_service): + print("inside test") diff --git a/tests/integration/test_psql.py b/tests/integration/test_psql.py index 4481df7..764ab97 100644 --- a/tests/integration/test_psql.py +++ b/tests/integration/test_psql.py @@ -2,10 +2,21 @@ from unittest import mock import subprocess import time -from features.step import FeaturesComputer, KafkaProducer, Step, Feature, FeatureVersion +from features.step import ( + FeaturesComputer, + CustomStreamHierarchicalExtractor, + KafkaProducer, + pd, + np, + Feature, + FeatureVersion, + Step, +) from db_plugins.db.sql.models import Object +import pytest +@pytest.mark.usefixtures("psql_service") class PSQLIntegrationTest(unittest.TestCase): @classmethod def setUpClass(self): @@ -70,8 +81,66 @@ def test_insert_step_metadata(self): self.assertEqual(len(self.step.db.query(Step).all()), 1) def test_insert_db(self): - oid = "oid" - features = {"test_1": 0} + oid = "ZTF1" + features = pd.DataFrame( + { + "oid": ["ZTF1"], + "feature_1": [123], + "feature_2": [456], + "not_a_feature": [-1], + } + ) + features.set_index("oid", inplace=True) + preprocess_id = "pre" + obj = Object(oid=oid) + self.step.db.session.add(obj) + self.step.db.session.commit() + self.step.insert_step_metadata() + self.step.config["STEP_METADATA"] = { + "STEP_ID": preprocess_id, + "STEP_NAME": "preprocess", + "STEP_VERSION": "test", + "STEP_COMMENTS": "", + "FEATURE_VERSION": "feature", + } + self.step.insert_step_metadata() + self.step.insert_feature_version(preprocess_id) + self.step.add_to_db(features) + self.assertEqual(len(self.step.db.query(Feature).all()), 3) + self.assertEqual(len(self.step.db.query(FeatureVersion).all()), 1) + + def test_insert_db_with_nan(self): + oid = "ZTF1" + features = pd.DataFrame( + { + "oid": ["ZTF1"], + "feature_1": [123], + "feature_2": [np.nan], + "not_a_feature": [-1], + } + ) + features.set_index("oid", inplace=True) + preprocess_id = "pre" + obj = Object(oid=oid) + self.step.db.session.add(obj) + self.step.db.session.commit() + self.step.insert_step_metadata() + self.step.config["STEP_METADATA"] = { + "STEP_ID": preprocess_id, + "STEP_NAME": "preprocess", + "STEP_VERSION": "test", + "STEP_COMMENTS": "", + "FEATURE_VERSION": "feature", + } + self.step.insert_step_metadata() + self.step.insert_feature_version(preprocess_id) + self.step.add_to_db(features) + self.assertEqual(len(self.step.db.query(Feature).all()), 3) + self.assertEqual(len(self.step.db.query(FeatureVersion).all()), 1) + + def test_insert_db_empty(self): + oid = "ZTF1" + features = pd.DataFrame() preprocess_id = "pre" obj = Object(oid=oid) self.step.db.session.add(obj) @@ -85,6 +154,7 @@ def test_insert_db(self): "FEATURE_VERSION": "feature", } self.step.insert_step_metadata() - self.step.insert_db(oid, features, preprocess_id) - self.assertEqual(len(self.step.db.query(Feature).all()), 1) + self.step.insert_feature_version(preprocess_id) + self.step.add_to_db(features) + self.assertEqual(len(self.step.db.query(Feature).all()), 0) self.assertEqual(len(self.step.db.query(FeatureVersion).all()), 1) diff --git a/tests/unittest/test_step.py b/tests/unittest/test_step.py index 48b980a..2740a38 100644 --- a/tests/unittest/test_step.py +++ b/tests/unittest/test_step.py @@ -1,13 +1,16 @@ import unittest import datetime from unittest import mock -from features.step import FeaturesComputer -from features.step import CustomStreamHierarchicalExtractor -from features.step import SQLConnection -from features.step import KafkaProducer -from features.step import pd -from features.step import np -from features.step import Feature, FeatureVersion +from features.step import ( + FeaturesComputer, + CustomStreamHierarchicalExtractor, + SQLConnection, + KafkaProducer, + pd, + np, + Feature, + FeatureVersion, +) from db_plugins.db.sql import SQLQuery @@ -31,141 +34,212 @@ def setUp(self): "STEP_NAME": "feature", "STEP_COMMENTS": "feature", "FEATURE_VERSION": "1.0-test", - } + }, } self.mock_database_connection = mock.create_autospec(SQLConnection) + self.mock_database_connection.engine = mock.Mock() self.mock_database_connection.session = mock.create_autospec(MockSession) - self.mock_custom_hierarchical_extractor = mock.create_autospec(FeaturesComputer) + self.mock_custom_hierarchical_extractor = mock.create_autospec( + CustomStreamHierarchicalExtractor + ) self.mock_producer = mock.create_autospec(KafkaProducer) self.step = FeaturesComputer( config=self.step_config, features_computer=self.mock_custom_hierarchical_extractor, db_connection=self.mock_database_connection, producer=self.mock_producer, - test_mode=True + test_mode=True, ) - @mock.patch.object(FeaturesComputer, "create_detections_dataframe") - def test_preprocess_detections(self, mock_create_dataframe): - detections = [{"oid": "oidtest", "candid": 123, "mjd": 1.0}] - detections_preprocessed = self.step.preprocess_detections(detections) - mock_create_dataframe.assert_called_with(detections) - - def test_preprocess_non_detections(self): - non_detections = [] - non_detections = self.step.preprocess_non_detections(non_detections) - self.assertIsInstance(non_detections, pd.DataFrame) - - def test_preprocess_xmatches(self): - xmatches = self.step.preprocess_xmatches("test") - self.assertEqual(xmatches, "test") - - def test_preprocess_metadata(self): - metadata = self.step.preprocess_metadata("test") - self.assertEqual(metadata, "test") - - def test_create_detections_dataframe(self): - detections = {"oid": "oidtest", "candid": 123, "mjd": 1.0, "alert.sgscore1": 1} - df = self.step.create_detections_dataframe(detections) - self.assertIsInstance(df, pd.DataFrame) - self.assertFalse("alert.sgscore1" in df.columns) - self.assertTrue("sgscore1" in df.columns) - self.assertEqual("oid", df.index.name) + def test_insert_step_metadata(self): + self.step.insert_step_metadata() + self.mock_database_connection.query().get_or_create.assert_called_once() def test_compute_features(self): detections = pd.DataFrame() non_detections = pd.DataFrame() + metadata = pd.DataFrame() + xmatches = pd.DataFrame() self.mock_custom_hierarchical_extractor.compute_features.return_value = ( pd.DataFrame() ) - features = self.step.compute_features(detections, non_detections, {}, {}) + features = self.step.compute_features( + detections, non_detections, metadata, xmatches + ) self.mock_custom_hierarchical_extractor.compute_features.assert_called_with( - detections, non_detections=non_detections, metadata={}, xmatches={} + detections, + non_detections=non_detections, + metadata=metadata, + xmatches=xmatches, ) self.assertIsInstance(features, pd.DataFrame) - def test_convert_nan(self): - d = {"di": {}, "x": 1, "y": np.nan} - d = self.step.convert_nan(d) - expected = {"di": {}, "x": 1, "y": None} - self.assertEqual(d, expected) - - def test_insert_db_doesnt_exist(self): - oid = "ZTF1" - features = {"testfeature_1": 1} - version = self.step_config["STEP_METADATA"]["FEATURE_VERSION"] - feature_id = self.step_config["STEP_METADATA"]["STEP_VERSION"] - preprocess_id = "correction" - mock_feature_version = mock.create_autospec(FeatureVersion) - mock_feature_version.version = version - mock_feature_version.value = 1 + @mock.patch.object(pd, "read_sql") + def test_get_on_db(self, read_sql): + features = pd.DataFrame() + self.step.get_on_db(features) + read_sql.assert_called_once() + + def test_insert_feature_version(self): self.mock_database_connection.query().get_or_create.return_value = ( - mock_feature_version, - False, + "instance", + "created", ) - self.step.insert_db(oid, features, preprocess_id) - self.mock_database_connection.query().get_or_create.assert_has_calls( - [ - mock.call( - filter_by={ - "version": version, - "step_id_feature": feature_id, - "step_id_preprocess": preprocess_id, - } - ), - mock.call( - filter_by={ - "oid": oid, - "name": "testfeature_1", - "fid": 1, - "version": version, - }, - value=1, - ), - ] + self.step.insert_feature_version("preprocess_id") + self.assertEqual(self.step.feature_version, "instance") + + @mock.patch.object(FeaturesComputer, "get_fid") + def test_update_db_empty(self, get_fid): + to_update = pd.DataFrame() + out_columns = ["oid", "name", "value"] + apply_get_fid = lambda x: get_fid(x) + self.step.update_db(to_update, out_columns, apply_get_fid) + self.step.db.engine.execute.assert_not_called() + + def test_update_db_not_empty(self): + to_update = pd.DataFrame( + { + "oid": ["ZTF1"], + "feature_1": [123], + "feature_2": [np.nan], + "not_a_feature": [-1], + } ) - self.mock_database_connection.query().update.assert_called_once() - self.mock_database_connection.session.commit.assert_called_once() - - def test_insert_db_exist(self): - oid = "ZTF1" - features = {"testfeature_1": 1} - version = self.step_config["STEP_METADATA"]["FEATURE_VERSION"] - feature_id = self.step_config["STEP_METADATA"]["STEP_VERSION"] - preprocess_id = "correction" - mock_feature_version = mock.create_autospec(FeatureVersion) - mock_feature_version.version = version - self.mock_database_connection.query().get_or_create.return_value = ( - mock_feature_version, - True, + to_update.set_index("oid", inplace=True) + out_columns = ["oid", "name", "value"] + apply_get_fid = lambda x: self.step.get_fid(x) + feature_version = mock.Mock() + feature_version.version = "test" + self.step.feature_version = feature_version + expected = [ + { + "_oid": "ZTF1", + "_name": "feature", + "_fid": 1, + "_value": 123, + "_version": "test", + }, + { + "_oid": "ZTF1", + "_name": "feature", + "_fid": 2, + "_value": None, + "_version": "test", + }, + { + "_oid": "ZTF1", + "_name": "not_a_feature", + "_fid": -99, + "_value": -1, + "_version": "test", + }, + ] + updated = self.step.update_db(to_update, out_columns, apply_get_fid) + self.step.db.engine.execute.assert_called() + self.assertEqual(updated, expected) + + @mock.patch.object(FeaturesComputer, "get_fid") + def test_insert_db_empty(self, get_fid): + to_insert = pd.DataFrame() + out_columns = ["oid", "name", "value"] + apply_get_fid = lambda x: get_fid(x) + feature_version = mock.Mock() + feature_version.version = "test" + self.step.feature_version = feature_version + self.step.update_db(to_insert, out_columns, apply_get_fid) + + def test_insert_db_not_empty(self): + to_insert = pd.DataFrame( + { + "oid": ["ZTF1"], + "feature_1": [123], + "feature_2": [456], + "not_a_feature": [-1], + } ) - self.step.insert_db(oid, features, preprocess_id) - self.mock_database_connection.query().get_or_create.assert_has_calls( - [ - mock.call( - filter_by={ - "version": version, - "step_id_feature": feature_id, - "step_id_preprocess": preprocess_id, - } - ), - mock.call( - filter_by={ - "oid": oid, - "name": "testfeature_1", - "fid": 1, - "version": version, - }, - value=1, - ), - ] + to_insert.set_index("oid", inplace=True) + out_columns = ["oid", "name", "value"] + apply_get_fid = lambda x: self.step.get_fid(x) + feature_version = mock.Mock() + feature_version.version = "test" + self.step.feature_version = feature_version + self.step.insert_db(to_insert, out_columns, apply_get_fid) + dict_to_insert = [ + { + "oid": "ZTF1", + "fid": 1, + "name": "feature", + "value": 123, + "version": "test", + }, + { + "oid": "ZTF1", + "fid": 2, + "name": "feature", + "value": 456, + "version": "test", + }, + { + "oid": "ZTF1", + "fid": -99, + "name": "not_a_feature", + "value": -1, + "version": "test", + }, + ] + self.step.db.query().bulk_insert.assert_called_with(dict_to_insert, Feature) + + @mock.patch.object(FeaturesComputer, "get_on_db") + @mock.patch.object(FeaturesComputer, "update_db") + @mock.patch.object(FeaturesComputer, "insert_db") + def test_add_to_db_update(self, insert_db, update_db, get_on_db): + result = pd.DataFrame( + { + "oid": ["ZTF1"], + "feature_1": [123], + "feature_2": [456], + "not_a_feature": [-1], + } ) - self.mock_database_connection.query().update.assert_not_called() - self.mock_database_connection.session.commit.assert_called_once() + result.set_index("oid", inplace=True) + get_on_db.return_value = ["ZTF1"] + out_columns = ["oid", "name", "value"] + apply_get_fid = lambda x: self.step.get_fid(x) + self.step.add_to_db(result) + update_db.assert_called_once() + insert_db.assert_not_called() - def test_insert_step_metadata(self): - self.step.insert_step_metadata() - self.step.db.query().get_or_create.assert_called_once() + @mock.patch.object(FeaturesComputer, "get_on_db") + @mock.patch.object(FeaturesComputer, "update_db") + @mock.patch.object(FeaturesComputer, "insert_db") + def test_add_to_db_insert(self, insert_db, update_db, get_on_db): + result = pd.DataFrame( + { + "oid": ["ZTF1"], + "feature_1": [123], + "feature_2": [456], + "not_a_feature": [-1], + } + ) + result.set_index("oid", inplace=True) + get_on_db.return_value = [] + out_columns = ["oid", "name", "value"] + apply_get_fid = lambda x: self.step.get_fid(x) + self.step.add_to_db(result) + insert_db.assert_called_once() + update_db.assert_not_called() + + @mock.patch.object(FeaturesComputer, "get_on_db") + @mock.patch.object(FeaturesComputer, "update_db") + @mock.patch.object(FeaturesComputer, "insert_db") + def test_add_to_db_empty(self, insert_db, update_db, get_on_db): + result = pd.DataFrame() + get_on_db.return_value = [] + out_columns = ["oid", "name", "value"] + apply_get_fid = lambda x: self.step.get_fid(x) + self.step.add_to_db(result) + insert_db.assert_not_called() + update_db.assert_not_called() def test_get_fid(self): feature = "W1" @@ -177,54 +251,66 @@ def test_get_fid(self): feature = "somefeature_2" fid = self.step.get_fid(feature) self.assertEqual(fid, 2) + feature = "something_else" + fid = self.step.get_fid(feature) + self.assertEqual(fid, -99) + feature = 123 + fid = self.step.get_fid(feature) + self.assertEqual(fid, -99) + + def test_check_feature_name(self): + name = self.step.check_feature_name("feature_1") + self.assertEqual(name, "feature") + name = self.step.check_feature_name("feature_one_two") + self.assertEqual(name, "feature_one_two") @mock.patch.object(FeaturesComputer, "compute_features") - def test_execute_less_than_6(self, mock_compute): - message = { - "oid": "ZTF1", - "detections": [{"candid": 123, "oid": "ZTF1", "mjd": 456}], - "non_detections": [], - "xmatches": "", - "metadata": {}, - "preprocess_step_id": "preprocess", - } - self.step.execute(message) - mock_compute.assert_not_called() + @mock.patch.object(FeaturesComputer, "produce") + @mock.patch.object(FeaturesComputer, "add_to_db") + @mock.patch.object(FeaturesComputer, "insert_feature_version") + def test_execute_with_producer(self, insert_feature_version, add_to_db, produce, compute_features): + messages = [ + { + "oid": "ZTF1", + "candid": 123, + "detections": [{"candid": 123, "oid": "ZTF1", "mjd": 456, "fid": 1}] + * 10, + "non_detections": [], + "xmatches": {"allwise": {"W1mag": 1, "W2mag": 2, "W3mag": 3}}, + "metadata": {"ps1": {"sgscore1": {}}}, + "preprocess_step_id": "preprocess", + } + ] + mock_feature_version = mock.create_autospec(FeatureVersion) + mock_feature_version.version = self.step_config["FEATURE_VERSION"] + self.step.execute(messages) + produce.assert_called_once() - @mock.patch.object(FeaturesComputer, "convert_nan") @mock.patch.object(FeaturesComputer, "compute_features") - def test_execute_no_features(self, mock_compute, mock_convert_nan): - message = { + @mock.patch.object(FeaturesComputer, "produce") + @mock.patch.object(FeaturesComputer, "add_to_db") + @mock.patch.object(FeaturesComputer, "insert_feature_version") + def test_execute_duplicates(self, insert_feature_version, add_to_db, produce, compute_features): + message1 = { "oid": "ZTF1", - "detections": [{"candid": 123, "oid": "ZTF1", "mjd": 456}], - "non_detections": [], - "xmatches": "", - "metadata": {}, + "candid": 123, + "detections": [{"candid": 123, "oid": "ZTF1", "mjd": 456, "fid": 1}] * 10, + "non_detections": [{"oid": "ZTF1", "fid": 1, "mjd": 456}], + "xmatches": {"allwise": {"W1mag": 1, "W2mag": 2, "W3mag": 3}}, + "metadata": {"ps1": {"sgscore1": {}}}, + "preprocess_step_id": "preprocess", } - mock_compute.return_value = pd.DataFrame() - self.step.execute(message) - mock_convert_nan.assert_not_called() - - @mock.patch.object(FeaturesComputer, "compute_features") - def test_execute_with_producer(self, mock_compute): - message = { + message2 = { "oid": "ZTF1", "candid": 123, "detections": [{"candid": 123, "oid": "ZTF1", "mjd": 456, "fid": 1}] * 10, - "non_detections": [], - "xmatches": {}, - "metadata": {}, + "non_detections": [{"oid": "ZTF1", "fid": 1, "mjd": 456}], + "xmatches": {"allwise": {"W1mag": 1, "W2mag": 2, "W3mag": 3}}, + "metadata": {"ps1": {"sgscore1": {}}}, "preprocess_step_id": "preprocess", } - df = pd.DataFrame({"oid": ["ZTF1"] * 10, "feat_1": [1] * 10}) - df.set_index("oid", inplace=True) - mock_compute.return_value = df + messages = [message1, message2] mock_feature_version = mock.create_autospec(FeatureVersion) mock_feature_version.version = self.step_config["FEATURE_VERSION"] - self.mock_database_connection.query().get_or_create.return_value = ( - mock_feature_version, - True, - ) - - self.step.execute(message) - self.mock_producer.produce.assert_called_once() + self.step.execute(messages) + produce.assert_called_once()