Merge pull request #86 from punch-mission/fix-tlm-comparison

make sure the path is extracted for tlm
punch-mission · Nov 30, 2024 · 17354e2 · 17354e2
2 parents 256d34d + d0bfba6
commit 17354e2
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 15 deletions.
diff --git a/punchpipe/control/db.py b/punchpipe/control/db.py
@@ -1,8 +1,8 @@
 import os
 
 from sqlalchemy import TEXT, Boolean, Column, DateTime, Float, Integer, String
-from sqlalchemy.orm import declarative_base
 from sqlalchemy.dialects.mysql import INTEGER
+from sqlalchemy.orm import declarative_base
 
 from punchpipe.error import MissingCCSDSDataError
 

diff --git a/punchpipe/flows/level0.py b/punchpipe/flows/level0.py
@@ -5,6 +5,8 @@
 import pandas as pd
 from ndcube import NDCube
 from prefect import flow
+from prefect.blocks.core import Block
+from prefect.blocks.fields import SecretDict
 from punchbowl.data import get_base_file_name
 from punchbowl.data.io import write_ndcube_to_fits
 from punchbowl.data.meta import NormalizedMetadata
@@ -26,8 +28,6 @@
 )
 from punchpipe.level0.meta import POSITIONS_TO_CODES, convert_pfw_position_to_polarizer
 
-from prefect.blocks.fields import SecretDict
-from prefect.blocks.core import Block
 
 class SpacecraftMapping(Block):
     mapping: SecretDict
@@ -63,7 +63,6 @@ def level0_form_images(session=None, pipeline_config_path=None):
         errors = []
 
         for t in distinct_times:
-            print(t)
             image_packets_entries = session.query(SciPacket).where(and_(SciPacket.timestamp == t[0],
                                                                 SciPacket.spacecraft_id == spacecraft[0])).all()
             image_compression = [unpack_compression_settings(packet.compression_settings)
@@ -98,15 +97,13 @@ def level0_form_images(session=None, pipeline_config_path=None):
                 try:
                     image = form_from_jpeg_compressed(ordered_image_content)
                 except ValueError:
-                    print("jpeg failed")
                     skip_image = True
                     error = {'start_time': image_packets_entries[0].timestamp.strftime("%Y-%m-%d %h:%m:%s"),
                              'start_block': image_packets_entries[0].flash_block,
                              'replay_length': image_packets_entries[-1].flash_block
                                               - image_packets_entries[0].flash_block}
                     errors.append(error)
             else:
-                print("not jpeg compressed")
                 skip_image = True
                 error = {'start_time': image_packets_entries[0].timestamp.strftime("%Y-%m-%d %h:%m:%s"),
                          'start_block': image_packets_entries[0].flash_block,
@@ -116,7 +113,6 @@ def level0_form_images(session=None, pipeline_config_path=None):
 
             # check the quality of the image
             if not skip_image and not image_is_okay(image, config):
-                print("image isn't okay")
                 skip_image = True
                 error = {'start_time': image_packets_entries[0].timestamp.strftime("%Y-%m-%d %h:%m:%s"),
                          'start_block': image_packets_entries[0].flash_block,
@@ -125,12 +121,9 @@ def level0_form_images(session=None, pipeline_config_path=None):
                 errors.append(error)
 
             if not skip_image:
-                print("NOT SKIPPING")
                 spacecraft_secrets = SpacecraftMapping.load("spacecraft-ids").mapping.get_secret_value()
                 moc_index = spacecraft_secrets["moc"].index(image_packets_entries[0].spacecraft_id)
                 spacecraft_id = spacecraft_secrets["soc"][moc_index]
-                # spacecraft_id = spacecraft_id_mapper[image_packets_entries[0].spacecraft_id]
-                print("TO HERE")
 
                 metadata_contents = get_fits_metadata(image_packets_entries[0].timestamp,
                                                       image_packets_entries[0].spacecraft_id,

diff --git a/punchpipe/level0/ccsds.py b/punchpipe/level0/ccsds.py
@@ -55,11 +55,9 @@ def process_telemetry_file(telemetry_file_path):
     apid_separated_tlm = open_and_split_packet_file(telemetry_file_path)
     parsed_data = {}
     for apid, stream in apid_separated_tlm.items():
-        print(apid)
         if apid not in PACKET_APID2NAME or apid in SKIP_APIDS:
-            print(f"skipping {apid}")
+            pass
         else:
-            print(apid, PACKET_APID2NAME[apid])
             definition = load_packet_def(PACKET_APID2NAME[apid])
             parsed_data[apid] = definition.load(stream, include_primary_header=True)
     return parsed_data

diff --git a/punchpipe/level0/core.py b/punchpipe/level0/core.py
@@ -18,7 +18,7 @@
 from punchpipe.control.db import ENGPFWPacket, EngXACTPacket, SciPacket, TLMFiles, get_closest_eng_packets
 from punchpipe.control.util import get_database_session
 from punchpipe.error import CCSDSPacketConstructionWarning, CCSDSPacketDatabaseUpdateWarning
-from punchpipe.level0.ccsds import PACKET_APID2NAME, process_telemetry_file, get_single_packet
+from punchpipe.level0.ccsds import PACKET_APID2NAME, get_single_packet, process_telemetry_file
 from punchpipe.level0.meta import eci_quaternion_to_ra_dec
 
 software_version = importlib.metadata.version("punchpipe")
@@ -44,7 +44,7 @@ def detect_new_tlm_files(pipeline_config: dict, session=None) -> [str]:
 
     tlm_directory = pipeline_config['tlm_directory']
     found_tlm_files = set(glob(tlm_directory + '/*.tlm'))
-    database_tlm_files = set(session.query(TLMFiles.path).distinct().all())
+    database_tlm_files = set([p[0] for p in session.query(TLMFiles.path).distinct().all()])
     return list(found_tlm_files - database_tlm_files)