diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 35694c1..764e126 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12"] + python-version: ["3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml deleted file mode 100644 index 422170c..0000000 --- a/.github/workflows/docs.yaml +++ /dev/null @@ -1,38 +0,0 @@ -name: deploy-docs - -# Only run this when the master branch changes -on: - push: - branches: - - main - -# This job installs dependencies, builds the book, and pushes it to `gh-pages` -jobs: - deploy-docs: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - # Install dependencies - - name: Set up Python 3.10 - uses: actions/setup-python@v2 - with: - python-version: "3.11" - - - name: Install dependencies - run: | - eval `ssh-agent -s` - ssh-add - <<< '${{ secrets.PRIVATE_SSH_KEY }}' - pip install ".[docs]" - - # Build the book - - name: Sphinx build - run: | - cd ./docs; make html; cd .. - - # Push the book's HTML to github-pages - - name: GitHub Pages action - uses: peaceiris/actions-gh-pages@v3.6.1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./docs/build/html diff --git a/.github/workflows/python-publish.yaml b/.github/workflows/python-publish.yaml new file mode 100644 index 0000000..fb5b6bf --- /dev/null +++ b/.github/workflows/python-publish.yaml @@ -0,0 +1,36 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + deploy: + runs-on: ubuntu-latest + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build -s + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..14d0a0a --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,33 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +formats: + - pdf + - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - method: pip + extra_requirements: + - all + - docs + path: . diff --git a/README.md b/README.md index e5f75a9..311b14b 100644 --- a/README.md +++ b/README.md @@ -1,43 +1,36 @@ # punchpipe + `punchpipe` is the data processing pipeline for [the PUNCH mission](https://punch.space.swri.edu/). All the science code and actual calibration functionality lives in `punchbowl`. This package only automates the control segment for the Science Operations Center. +> [!CAUTION] +> This package is still being developed. There will be breaking code changes until v1. +> We advise you to wait until then to use it. + The `punchpipe` is organized into segments, i.e. levels of processing to produce specific data products. Segments are referred in code by their ending level, e.g. `level1` means the Level 0 to Level 1 segment. ## Accessing the data +Coming soon. + ## First-time setup -1. Create a clean virtual environment. You can do this with conda using `conda env create --name ENVIRONMENT-NAME` -2. Install `punchbowl` using `pip install .` in the `punchbowl` directory. -3. Install `punchpipe` using `pip install .` while in this directory -4. Set up database credentials Prefect block by running `python scripts/credentials.py`. - - If this file does not exist for you. You need to determine your mySQL credentials then create a block in Python: - ```py - from punchpipe.controlsegment.db import MySQLCredentials - cred = MySQLCredentials(user="username", password="password") - cred.save('mysql-cred') - ``` -5. Set up databases by running `scripts/create_db.py` directory. -6. Build all the necessary deployments for Prefect by following [these instructions](https://docs.prefect.io/concepts/deployments/). - - See below for an example: - ```shell - ./deploy.sh - ``` -7. Create a work queue in the Prefect UI for the deployments (will need to run `prefect orion start` to get the UI) -8. Create an agent for the work queue by following instructions in the UI + +Coming soon. ## Running -1. Make sure first-time setup is complete -2. Launch Prefect using `prefect orion start` -3. Create agents for the work queues by following the instructions in the UI for the work queue -## Resetting -1. Reset the Prefect Orion database using `prefect orion database reset`. -2. Remove all the `punchpipe` databases by running `erase_db.sql` +Coming soon. + +## Getting help + +Please open an issue or discussion on this repo. ## Contributing -## Licensing +We encourage all contributions. +If you have a problem with the code or would like to see a new feature, please open an issue. +Or you can submit a pull request. + diff --git a/docs/source/conf.py b/docs/source/conf.py index deb32cc..dc86360 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,3 +1,6 @@ +from importlib.metadata import version as get_version +from packaging.version import Version + # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full @@ -16,13 +19,16 @@ # -- Project information ----------------------------------------------------- - -project = 'punchpipe' -copyright = '2023, PUNCH Science Operations Center' -author = 'PUNCH Science Operations Center' +project = "punchpipe" +copyright = "2024, PUNCH Science Operations Center" +author = "PUNCH Science Operations Center" # The full version, including alpha/beta/rc tags -release = '0.0.1' +release: str = get_version("punchpipe") +version: str = release +_version = Version(release) +if _version.is_devrelease: + version = release = f"{_version.base_version}.dev{_version.dev}" # -- General configuration --------------------------------------------------- diff --git a/punchpipe/controlsegment/db.py b/punchpipe/controlsegment/db.py index 7b0a154..b460a1b 100644 --- a/punchpipe/controlsegment/db.py +++ b/punchpipe/controlsegment/db.py @@ -1,6 +1,6 @@ import os -from sqlalchemy import TEXT, Column, DateTime, Integer, String, Boolean, JSON +from sqlalchemy import TEXT, Column, DateTime, Integer, String, Boolean from sqlalchemy.orm import declarative_base Base = declarative_base() diff --git a/punchpipe/controlsegment/util.py b/punchpipe/controlsegment/util.py index 27a321a..7fc00b1 100644 --- a/punchpipe/controlsegment/util.py +++ b/punchpipe/controlsegment/util.py @@ -8,7 +8,7 @@ from prefect_sqlalchemy import SqlAlchemyConnector from ndcube import NDCube from punchbowl.data import write_ndcube_to_fits, get_base_file_name -from sqlalchemy import and_, or_ +from sqlalchemy import or_ from punchpipe.controlsegment.db import File diff --git a/punchpipe/deliver.py b/punchpipe/deliver.py deleted file mode 100644 index 33f67a9..0000000 --- a/punchpipe/deliver.py +++ /dev/null @@ -1,56 +0,0 @@ -import hashlib -import os -from zipfile import ZipFile -import time -from datetime import datetime, timedelta - -import pandas as pd -from prefect import task, flow - -from punchpipe.controlsegment.util import get_database_session, get_files_in_time_window - - -def hash_one_file(path): - file_hash = hashlib.sha256() - with open(path, 'rb') as f: - fb = f.read() - file_hash.update(fb) - - return file_hash.hexdigest() - -@task -def build_noaa_manifest(file_paths): - hashes = [hash_one_file(file_path) for file_path in file_paths] - return pd.DataFrame({'file': file_paths, 'hash': hashes}) - - -@task -def get_noaa_files_in_time_range(start_time, end_time, session=None): - if session is None: - session = get_database_session() - - paths = [] - for obs_code in ["1", "2", "3", "4"]: - paths += get_files_in_time_window("Q", "CR", obs_code, start_time, end_time, session=session) - return paths - - -@flow -def create_noaa_delivery(time_window=timedelta(hours=1)): - end_time = datetime.now() - start_time = end_time - time_window - timestamp = end_time.strftime('%Y%m%d%H%M%S') - - file_paths = get_noaa_files_in_time_range(start_time, end_time) - - manifest = build_noaa_manifest(file_paths) - manifest_path = f'manifest_{timestamp}.csv' - manifest.to_csv(manifest_path) - - zip_path = f'noaa_{timestamp}.zip' - with ZipFile(zip_path, 'w') as zip_object: - for file_path in file_paths: - zip_object.write(file_path) - zip_object.write(manifest_path) - - return zip_path diff --git a/punchpipe/flows/level2.py b/punchpipe/flows/level2.py index d4efbb3..2c4da64 100644 --- a/punchpipe/flows/level2.py +++ b/punchpipe/flows/level2.py @@ -5,7 +5,6 @@ from prefect import flow, task, get_run_logger from punchbowl.level2.flow import level2_core_flow -from sqlalchemy import and_ from punchpipe import __version__ from punchpipe.controlsegment.db import File, Flow diff --git a/punchpipe/flows/levelq.py b/punchpipe/flows/levelq.py index c3c2029..67a5854 100644 --- a/punchpipe/flows/levelq.py +++ b/punchpipe/flows/levelq.py @@ -4,8 +4,7 @@ from datetime import datetime from prefect import flow, task, get_run_logger -from punchbowl.level2.flow import level2_core_flow, levelq_core_flow -from sqlalchemy import and_ +from punchbowl.level2.flow import levelq_core_flow from punchpipe import __version__ from punchpipe.controlsegment.db import File, Flow diff --git a/punchpipe/level0/ccsds.py b/punchpipe/level0/ccsds.py index e38b72e..cd2913a 100644 --- a/punchpipe/level0/ccsds.py +++ b/punchpipe/level0/ccsds.py @@ -2,7 +2,10 @@ import os import ccsdspy +import numpy as np from ccsdspy.utils import split_by_apid +from matplotlib import pyplot as plt +import pylibjpeg PACKET_NAME2APID = { "ENG_LZ": 0x60, @@ -50,12 +53,79 @@ def process_telemetry_file(telemetry_file_path): apid_separated_tlm = open_and_split_packet_file(telemetry_file_path) parsed_data = {} for apid, stream in apid_separated_tlm.items(): - definition = load_packet_def(PACKET_APID2NAME[apid]) - parsed_data[apid] = definition.load(stream, include_primary_header=True) + if apid not in PACKET_APID2NAME or apid in [96]: + print(f"skipping {apid}") + else: + print(apid, PACKET_APID2NAME[apid]) + definition = load_packet_def(PACKET_APID2NAME[apid]) + parsed_data[apid] = definition.load(stream, include_primary_header=True) return parsed_data +def parse_compression_settings(values): + # return [{'test': bool(v & 1), 'jpeg': bool(v & 2), 'sqrt': bool(v & 4)} for v in values] + return [{'test': bool(v & 0b1000000000000000), + 'jpeg': bool(v & 0b0100000000000000), + 'sqrt': bool(v & 0b0010000000000000)} for v in values] + + +def unpack_compression_settings(com_set_val: "bytes|int"): + """Unpack image compression control register value. + + See `SciPacket.COMPRESSION_REG` for details.""" + + if isinstance(com_set_val, bytes): + assert len(com_set_val) == 2, f"Compression settings should be a 2-byte field, got {len(com_set_val)} bytes" + compress_config = int.from_bytes(com_set_val, "big") + elif isinstance(com_set_val, (int, np.integer)): + assert com_set_val <= 0xFFFF, f"Compression settings should fit within 2 bytes, got \\x{com_set_val:X}" + compress_config = int(com_set_val) + else: + raise TypeError + settings_dict = {"SCALE": compress_config >> 8, + "RSVD": (compress_config >> 7) & 0b1, + "PMB_INIT": (compress_config >> 6) & 0b1, + "CMP_BYP": (compress_config >> 5) & 0b1, + "BSEL": (compress_config >> 3) & 0b11, + "SQRT": (compress_config >> 2) & 0b1, + "JPEG": (compress_config >> 1) & 0b1, + "TEST": compress_config & 0b1} + return settings_dict + + +def unpack_acquisition_settings(acq_set_val: "bytes|int"): + """Unpack CEB image acquisition register value. + + See `SciPacket.ACQUISITION_REG` for details.""" + + if isinstance(acq_set_val, bytes): + assert len(acq_set_val) == 4, f"Acquisition settings should be a 4-byte field, got {len(acq_set_val)} bytes" + acquire_config = int.from_bytes(acq_set_val, "big") + elif isinstance(acq_set_val, (int, np.integer)): + assert acq_set_val <= 0xFFFFFFFF, f"Acquisition settings should fit within 4 bytes, got \\x{acq_set_val:X}" + acquire_config = int(acq_set_val) + else: + raise TypeError + settings_dict = {"DELAY": acquire_config >> 24, + "IMG_NUM": (acquire_config >> 21) & 0b111, + "EXPOSURE": (acquire_config >> 8) & 0x1FFF, + "TABLE1": (acquire_config >> 4) & 0b1111, + "TABLE2": acquire_config & 0b1111} + return settings_dict + + if __name__ == "__main__": - path = "/Users/jhughes/Desktop/sdf/punchbowl/Level0/packets/2024-02-09/PUNCH_NFI00_RAW_2024_040_21_32_V01.tlm" + path = "/Users/jhughes/Desktop/data/PUNCH_CCSDS/RAW_CCSDS_DATA/PUNCH_NFI00_RAW_2024_160_19_37_V01.tlm" + # path = "/Users/jhughes/Desktop/data/PUNCH_CCSDS/RAW_CCSDS_DATA/PUNCH_WFI01_RAW_2024_117_22_00_V01.tlm" parsed = process_telemetry_file(path) - print(parsed[0x20]) + print(parse_compression_settings(parsed[0x20]['SCI_XFI_COM_SET'])[22:44]) + + fig, ax = plt.subplots() + ax.plot(parsed[0x20]['CCSDS_PACKET_LENGTH']) + plt.show() + + print(parsed[0x20]['CCSDS_PACKET_LENGTH'][22:44]) + + img = np.concatenate(parsed[0x20]['SCI_XFI_IMG_DATA'][22:44]) + img = pylibjpeg.decode(img.tobytes()) + diff --git a/punchpipe/level0/flow.py b/punchpipe/level0/flow.py index 3443a3a..75bc2eb 100644 --- a/punchpipe/level0/flow.py +++ b/punchpipe/level0/flow.py @@ -1,6 +1,5 @@ import json import base64 -import uuid from datetime import datetime, timedelta import warnings @@ -13,7 +12,7 @@ from punchpipe.level0.ccsds import process_telemetry_file, PACKET_APID2NAME, unpack_compression_settings from punchpipe.controlsegment.db import SciPacket, EngPacket -from punchpipe.controlsegment.util import (get_database_session, load_pipeline_configuration) +from punchpipe.controlsegment.util import (get_database_session) from punchpipe.error import CCSDSPacketConstructionWarning, CCSDSPacketDatabaseUpdateWarning diff --git a/pyproject.toml b/pyproject.toml index 26e3e57..78180e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [build-system] -requires = ["setuptools", - "wheel"] +requires = ["setuptools>=64", "setuptools-scm>=8", "wheel"] +build-backend = "setuptools.build_meta" [project] name = "punchpipe" @@ -9,7 +9,7 @@ version = "0.0.1" dependencies = [ "click", "ccsdspy", - "punchbowl @ git+ssh://git@github.com/punch-mission/punchbowl@main", + "punchbowl", "prefect[sqlalchemy]", "pymysql", "pandas", @@ -51,7 +51,8 @@ test = ["pre-commit", "pytest-mock-resources[mysql]", "freezegun", "ruff"] -docs = ["sphinx", +docs = ["packaging", + "sphinx", "pydata-sphinx-theme", "sphinx-autoapi", "sphinx-favicon", @@ -69,6 +70,8 @@ Repository = "https://github.com/punch-mission/punchpipe.git" "Bug Tracker" = "https://github.com/punch-mission/punchpipe/issues" #Changelog = "https://github.com/punch-mission/thuban/blob/main/CHANGELOG.md" +[tool.setuptools_scm] + [tool.setuptools] packages = ["punchpipe"] diff --git a/scripts/erase_db.sql b/scripts/erase_db.sql index 65a0ab5..cb7d625 100644 --- a/scripts/erase_db.sql +++ b/scripts/erase_db.sql @@ -6,3 +6,6 @@ USE punchpipe; DROP TABLE IF EXISTS relationships; DROP TABLE IF EXISTS files; DROP TABLE IF EXISTS flows; +DROP TABLE IF EXISTS packets; +DROP TABLE IF EXISTS sci_packets; +DROP TABLE IF EXISTS eng_packets;