From 1eadaae3d2338e3c2dc0276d4b54c52dfb58e68d Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Sat, 30 Nov 2024 10:07:20 -0500 Subject: [PATCH 1/3] make sure the path is extracted in tlm --- punchpipe/level0/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/punchpipe/level0/core.py b/punchpipe/level0/core.py index 8e3a90b..8efdd4f 100644 --- a/punchpipe/level0/core.py +++ b/punchpipe/level0/core.py @@ -44,7 +44,7 @@ def detect_new_tlm_files(pipeline_config: dict, session=None) -> [str]: tlm_directory = pipeline_config['tlm_directory'] found_tlm_files = set(glob(tlm_directory + '/*.tlm')) - database_tlm_files = set(session.query(TLMFiles.path).distinct().all()) + database_tlm_files = set([p[0] for p in session.query(TLMFiles.path).distinct().all()]) return list(found_tlm_files - database_tlm_files) From 3accddfd0b434af5d4794d0caea2f3602063f14f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 30 Nov 2024 15:09:36 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- punchpipe/control/db.py | 2 +- punchpipe/flows/level0.py | 4 ++-- punchpipe/level0/core.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/punchpipe/control/db.py b/punchpipe/control/db.py index 8286d1d..69c7890 100644 --- a/punchpipe/control/db.py +++ b/punchpipe/control/db.py @@ -1,8 +1,8 @@ import os from sqlalchemy import TEXT, Boolean, Column, DateTime, Float, Integer, String -from sqlalchemy.orm import declarative_base from sqlalchemy.dialects.mysql import INTEGER +from sqlalchemy.orm import declarative_base from punchpipe.error import MissingCCSDSDataError diff --git a/punchpipe/flows/level0.py b/punchpipe/flows/level0.py index 9fbd6e1..c0c9c7b 100644 --- a/punchpipe/flows/level0.py +++ b/punchpipe/flows/level0.py @@ -5,6 +5,8 @@ import pandas as pd from ndcube import NDCube from prefect import flow +from prefect.blocks.core import Block +from prefect.blocks.fields import SecretDict from punchbowl.data import get_base_file_name from punchbowl.data.io import write_ndcube_to_fits from punchbowl.data.meta import NormalizedMetadata @@ -26,8 +28,6 @@ ) from punchpipe.level0.meta import POSITIONS_TO_CODES, convert_pfw_position_to_polarizer -from prefect.blocks.fields import SecretDict -from prefect.blocks.core import Block class SpacecraftMapping(Block): mapping: SecretDict diff --git a/punchpipe/level0/core.py b/punchpipe/level0/core.py index 8efdd4f..584970e 100644 --- a/punchpipe/level0/core.py +++ b/punchpipe/level0/core.py @@ -18,7 +18,7 @@ from punchpipe.control.db import ENGPFWPacket, EngXACTPacket, SciPacket, TLMFiles, get_closest_eng_packets from punchpipe.control.util import get_database_session from punchpipe.error import CCSDSPacketConstructionWarning, CCSDSPacketDatabaseUpdateWarning -from punchpipe.level0.ccsds import PACKET_APID2NAME, process_telemetry_file, get_single_packet +from punchpipe.level0.ccsds import PACKET_APID2NAME, get_single_packet, process_telemetry_file from punchpipe.level0.meta import eci_quaternion_to_ra_dec software_version = importlib.metadata.version("punchpipe") From 63ad10e344d967da6aa76997609bbc17cc354f73 Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Sat, 30 Nov 2024 10:51:30 -0500 Subject: [PATCH 3/3] remove old prints --- punchpipe/flows/level0.py | 7 ------- punchpipe/level0/ccsds.py | 4 +--- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/punchpipe/flows/level0.py b/punchpipe/flows/level0.py index 9fbd6e1..44d456f 100644 --- a/punchpipe/flows/level0.py +++ b/punchpipe/flows/level0.py @@ -63,7 +63,6 @@ def level0_form_images(session=None, pipeline_config_path=None): errors = [] for t in distinct_times: - print(t) image_packets_entries = session.query(SciPacket).where(and_(SciPacket.timestamp == t[0], SciPacket.spacecraft_id == spacecraft[0])).all() image_compression = [unpack_compression_settings(packet.compression_settings) @@ -98,7 +97,6 @@ def level0_form_images(session=None, pipeline_config_path=None): try: image = form_from_jpeg_compressed(ordered_image_content) except ValueError: - print("jpeg failed") skip_image = True error = {'start_time': image_packets_entries[0].timestamp.strftime("%Y-%m-%d %h:%m:%s"), 'start_block': image_packets_entries[0].flash_block, @@ -106,7 +104,6 @@ def level0_form_images(session=None, pipeline_config_path=None): - image_packets_entries[0].flash_block} errors.append(error) else: - print("not jpeg compressed") skip_image = True error = {'start_time': image_packets_entries[0].timestamp.strftime("%Y-%m-%d %h:%m:%s"), 'start_block': image_packets_entries[0].flash_block, @@ -116,7 +113,6 @@ def level0_form_images(session=None, pipeline_config_path=None): # check the quality of the image if not skip_image and not image_is_okay(image, config): - print("image isn't okay") skip_image = True error = {'start_time': image_packets_entries[0].timestamp.strftime("%Y-%m-%d %h:%m:%s"), 'start_block': image_packets_entries[0].flash_block, @@ -125,12 +121,9 @@ def level0_form_images(session=None, pipeline_config_path=None): errors.append(error) if not skip_image: - print("NOT SKIPPING") spacecraft_secrets = SpacecraftMapping.load("spacecraft-ids").mapping.get_secret_value() moc_index = spacecraft_secrets["moc"].index(image_packets_entries[0].spacecraft_id) spacecraft_id = spacecraft_secrets["soc"][moc_index] - # spacecraft_id = spacecraft_id_mapper[image_packets_entries[0].spacecraft_id] - print("TO HERE") metadata_contents = get_fits_metadata(image_packets_entries[0].timestamp, image_packets_entries[0].spacecraft_id, diff --git a/punchpipe/level0/ccsds.py b/punchpipe/level0/ccsds.py index c1d7d96..26a4498 100644 --- a/punchpipe/level0/ccsds.py +++ b/punchpipe/level0/ccsds.py @@ -55,11 +55,9 @@ def process_telemetry_file(telemetry_file_path): apid_separated_tlm = open_and_split_packet_file(telemetry_file_path) parsed_data = {} for apid, stream in apid_separated_tlm.items(): - print(apid) if apid not in PACKET_APID2NAME or apid in SKIP_APIDS: - print(f"skipping {apid}") + pass else: - print(apid, PACKET_APID2NAME[apid]) definition = load_packet_def(PACKET_APID2NAME[apid]) parsed_data[apid] = definition.load(stream, include_primary_header=True) return parsed_data