diff --git a/punchpipe/control/db.py b/punchpipe/control/db.py index 8286d1d..69c7890 100644 --- a/punchpipe/control/db.py +++ b/punchpipe/control/db.py @@ -1,8 +1,8 @@ import os from sqlalchemy import TEXT, Boolean, Column, DateTime, Float, Integer, String -from sqlalchemy.orm import declarative_base from sqlalchemy.dialects.mysql import INTEGER +from sqlalchemy.orm import declarative_base from punchpipe.error import MissingCCSDSDataError diff --git a/punchpipe/flows/level0.py b/punchpipe/flows/level0.py index 9fbd6e1..6642111 100644 --- a/punchpipe/flows/level0.py +++ b/punchpipe/flows/level0.py @@ -5,6 +5,8 @@ import pandas as pd from ndcube import NDCube from prefect import flow +from prefect.blocks.core import Block +from prefect.blocks.fields import SecretDict from punchbowl.data import get_base_file_name from punchbowl.data.io import write_ndcube_to_fits from punchbowl.data.meta import NormalizedMetadata @@ -26,8 +28,6 @@ ) from punchpipe.level0.meta import POSITIONS_TO_CODES, convert_pfw_position_to_polarizer -from prefect.blocks.fields import SecretDict -from prefect.blocks.core import Block class SpacecraftMapping(Block): mapping: SecretDict @@ -63,7 +63,6 @@ def level0_form_images(session=None, pipeline_config_path=None): errors = [] for t in distinct_times: - print(t) image_packets_entries = session.query(SciPacket).where(and_(SciPacket.timestamp == t[0], SciPacket.spacecraft_id == spacecraft[0])).all() image_compression = [unpack_compression_settings(packet.compression_settings) @@ -98,7 +97,6 @@ def level0_form_images(session=None, pipeline_config_path=None): try: image = form_from_jpeg_compressed(ordered_image_content) except ValueError: - print("jpeg failed") skip_image = True error = {'start_time': image_packets_entries[0].timestamp.strftime("%Y-%m-%d %h:%m:%s"), 'start_block': image_packets_entries[0].flash_block, @@ -106,7 +104,6 @@ def level0_form_images(session=None, pipeline_config_path=None): - image_packets_entries[0].flash_block} errors.append(error) else: - print("not jpeg compressed") skip_image = True error = {'start_time': image_packets_entries[0].timestamp.strftime("%Y-%m-%d %h:%m:%s"), 'start_block': image_packets_entries[0].flash_block, @@ -116,7 +113,6 @@ def level0_form_images(session=None, pipeline_config_path=None): # check the quality of the image if not skip_image and not image_is_okay(image, config): - print("image isn't okay") skip_image = True error = {'start_time': image_packets_entries[0].timestamp.strftime("%Y-%m-%d %h:%m:%s"), 'start_block': image_packets_entries[0].flash_block, @@ -125,12 +121,9 @@ def level0_form_images(session=None, pipeline_config_path=None): errors.append(error) if not skip_image: - print("NOT SKIPPING") spacecraft_secrets = SpacecraftMapping.load("spacecraft-ids").mapping.get_secret_value() moc_index = spacecraft_secrets["moc"].index(image_packets_entries[0].spacecraft_id) spacecraft_id = spacecraft_secrets["soc"][moc_index] - # spacecraft_id = spacecraft_id_mapper[image_packets_entries[0].spacecraft_id] - print("TO HERE") metadata_contents = get_fits_metadata(image_packets_entries[0].timestamp, image_packets_entries[0].spacecraft_id, diff --git a/punchpipe/level0/ccsds.py b/punchpipe/level0/ccsds.py index c1d7d96..26a4498 100644 --- a/punchpipe/level0/ccsds.py +++ b/punchpipe/level0/ccsds.py @@ -55,11 +55,9 @@ def process_telemetry_file(telemetry_file_path): apid_separated_tlm = open_and_split_packet_file(telemetry_file_path) parsed_data = {} for apid, stream in apid_separated_tlm.items(): - print(apid) if apid not in PACKET_APID2NAME or apid in SKIP_APIDS: - print(f"skipping {apid}") + pass else: - print(apid, PACKET_APID2NAME[apid]) definition = load_packet_def(PACKET_APID2NAME[apid]) parsed_data[apid] = definition.load(stream, include_primary_header=True) return parsed_data diff --git a/punchpipe/level0/core.py b/punchpipe/level0/core.py index 8e3a90b..584970e 100644 --- a/punchpipe/level0/core.py +++ b/punchpipe/level0/core.py @@ -18,7 +18,7 @@ from punchpipe.control.db import ENGPFWPacket, EngXACTPacket, SciPacket, TLMFiles, get_closest_eng_packets from punchpipe.control.util import get_database_session from punchpipe.error import CCSDSPacketConstructionWarning, CCSDSPacketDatabaseUpdateWarning -from punchpipe.level0.ccsds import PACKET_APID2NAME, process_telemetry_file, get_single_packet +from punchpipe.level0.ccsds import PACKET_APID2NAME, get_single_packet, process_telemetry_file from punchpipe.level0.meta import eci_quaternion_to_ra_dec software_version = importlib.metadata.version("punchpipe") @@ -44,7 +44,7 @@ def detect_new_tlm_files(pipeline_config: dict, session=None) -> [str]: tlm_directory = pipeline_config['tlm_directory'] found_tlm_files = set(glob(tlm_directory + '/*.tlm')) - database_tlm_files = set(session.query(TLMFiles.path).distinct().all()) + database_tlm_files = set([p[0] for p in session.query(TLMFiles.path).distinct().all()]) return list(found_tlm_files - database_tlm_files)