Merge pull request #316 from punch-mission/user-guide

PUNCH user guide

docs/source/conf.py

@@ -1,3 +1,5 @@
+import os
+import csv
 from importlib.metadata import version as get_version
 
 from packaging.version import Version
@@ -32,6 +34,37 @@
 if _version.is_devrelease:
     version = release = f"{_version.base_version}.dev{_version.dev}"
 
+
+# -- Metadata documentation ---------------------------------------------------
+
+def copy_and_truncate_csv(src_path, dest_path, columns, filter_column, filter_value, default_key):
+    with open(src_path, mode='r', newline='') as src_file:
+        reader = csv.DictReader(src_file)
+        filtered_rows = [
+            {
+                **{col: row[col] for col in columns},
+                'COMMENT': row['DEFAULT'] if not row['COMMENT'] else row['COMMENT']
+            } if row[filter_column] == filter_value else None
+            for row in reader
+        ]
+        filtered_rows = [row for row in filtered_rows if row]  # Remove None entries
+
+    os.makedirs(os.path.dirname(dest_path), exist_ok=True)
+    with open(dest_path, mode='w', newline='') as dest_file:
+        writer = csv.DictWriter(dest_file, fieldnames=columns)
+        writer.writeheader()
+        writer.writerows(filtered_rows)
+
+filter_column = 'TYPE'
+filter_value = 'keyword'
+
+src_csv_path = os.path.abspath(os.path.join('../../punchbowl/data/data/', 'omniheader.csv'))
+dest_csv_path = os.path.abspath(os.path.join('./data/', 'omniheader_select.csv'))
+columns_to_include = ['KEYWORD', 'COMMENT']
+
+copy_and_truncate_csv(src_csv_path, dest_csv_path, columns_to_include, filter_column, filter_value, 'DEFAULT')
+
+
 # -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be

docs/source/data/access.rst

@@ -1,2 +1,41 @@
 Accessing PUNCH Data
 ====================
+
+Downloading Data
+----------------
+Data output from the PUNCH data processing pipeline will be stored and accessible through the Solar Data Analysis Center (SDAC) - a portal for hosting through tools such as the Virtual Solar Observatory (VSO). From here PUNCH data products can be queried and requested for download using metadata within the data products.
+
+PUNCH data will also be accessible using the helioviewer tool, where it can be quickly visualized and stitched together with other observations for context.
+
+Reading Data
+------------
+Standard PUNCH data is stored as a standards-compliant FITS file, which bundles the primary data along with secondary data and metadata fully describing the observation. Each file is named with a convention that uniquely identifies the product - a sample being 'PUNCH_L3_PAM_20230704000000_v1.fits' - where L3 defines the data level, PAM is an example of a particular data product code, 20230704000000 is a timestamp in the format yyyyhhmmhhmmss, and _v1 is the version of the data (used in reprocessing).
+
+For most end-users the primary data of interest are PAM (low-noise full frame data gathered over one full spacecraft rotation cycle) and PTM (high-cadence trefoil mosaics).
+
+PUNCH FITS files are RICE compressed, reducing the overall file size while preserving data fidelity. Due to this compression, the zeroth HDU of each output data file contains information about the compression scheme. The first HDU (hdul[1]) contains the primary data array, along with an astropy header string describing that data. The second HDU (hdul[2]) contains the uncertainty array - corresponding on a pixel-by-pixel basis with the primary data array.
+
+These data are compatible with standard astropy FITS libraries, and can be read in as following the example,
+
+.. code-block:: python
+
+    filename = 'example_data/PUNCH_L3_PAM_20240620000000.fits'
+
+    with fits.open(filename) as hdul:
+        data = hdul[1].data
+        header = hdul[1].header
+        uncertainty = hdul[2].data
+
+These data can also be bundled together as an NDCube object, either manually or using some of the bundled IO tools within punchbowl.
+
+Data Projections
+----------------
+The PUNCH WFI instruments extend their field of view out to around 45-degrees from the Sun, creating a meshed virtual observatory extending to a diameter of nearly 180 solar radii. The wide nature of this field of view requires attention to the data projection being used for these data.
+
+For NFI data, the standard projection is a Gnomonic (TAN) coordinate system with distortion, a standard system employed for many data peering closer towards the sun.
+
+For individual WFI data, an azimuthal perspective (AZP) coordinate system with distortion is used.
+
+For full mosaics that combine data from WFI and NFI, an azimuthal equidistant (ARC) coordinate system is used, with data from each spacecraft frame aligned and projected to this standardized frame.
+
+Each data contains a set of World Coordinate System (WCS) parameters that describe the coordinates of the data, in both a helioprojective and celestial frame.

docs/source/data/data_codes.rst

@@ -0,0 +1,151 @@
+Data Product Codes
+====================
+
+PUNCH data products are organized into data processing levels from Level 0 (raw camera data) to Level 3 (calibrated science data). Within and across levels distinct data products and calibration files are marked with a unique identifying product code. For data in the spacecraft frame, each spacecraft is marked with a unique numeral identity - 1,2,3 for each WFI, or 4 for NFI.
+
+Data Product Codes
+------------------
+
+.. list-table::
+   :header-rows: 1
+
+   * - Level
+     - Code
+     - Description
+   * - 0
+     - PM1, PM2, PM3, PM4, PZ1, PZ2, PZ3, PZ4, PP1, PP2, PP3, PP4, CR1, CR2, CR3, CR4
+     - Science images in the standard polarization (PM, PZ, PP) and clear (CR) states
+   * - 0
+     - PX1, PX2, PX3, PX4
+     - Science images in a nonstandard polarization state
+   * - 0
+     - ST
+     - STEAM data packet (CSV format)
+   * - 1
+     - PM1, PM2, PM3, PM4, PZ1, PZ2, PZ3, PZ4, PP1, PP2, PP3, PP4, CR1, CR2, CR3, CR4
+     - Science images in the standard polarization (PM, PZ, PP) and clear (CR) states (photometrically calibrated)
+   * - 2
+     - PTM
+     - Pol. science mosaics (Trefoil) in output coordinates, resolved into MZP pol. triplets, and uncertainty layer
+   * - 2
+     - CTM
+     - Clear science mosaics (Trefoil) in output coordinates, resolved into image and uncertainty layer
+   * - 2
+     - PNN
+     - Pol. NFI images in output coordinates, resolved into MZP pol. triplets, and uncertainty layer
+   * - 2
+     - CNN
+     - Clear NFI images in output coordinates, resolved into image, and uncertainty layer
+   * - Q
+     - CNN
+     - QuickPUNCH NFI images
+   * - Q
+     - CTM
+     - QuickPUNCH Mosaic images (5.4–80 Rsun)
+   * - L
+     - CNN
+     - Quicklook Clear NFI images
+   * - L
+     - PNN
+     - Quicklook Polarized NFI images
+   * - L
+     - CTM
+     - Quicklook Clear Mosaic images
+   * - L
+     - PTM
+     - Quicklook Polarized Mosaic images
+   * - 3
+     - CAM
+     - Clear low-noise science mosaic, bkg-sub & resolved into B & uncertainty layer
+   * - 3
+     - PAN
+     - Polarized low-noise NFI science image, bkg-sub & resolved into B, pB, & uncertainty layer
+   * - 3
+     - CAN
+     - Clear low-noise NFI science image, bkg-sub & resolved into B & uncertainty layer
+   * - 3
+     - PTM
+     - Polarized K PUNCH science mosaics (Trefoil), bkg-sub & resolved into B, pB, & uncertainty layer
+   * - 3
+     - CTM
+     - Clear K science mosaics (Trefoil), bkg-sub & resolved into B & uncertainty layer
+   * - 3
+     - PNN
+     - Polarized K NFI science image, bkg-sub & resolved into B, pB, & uncertainty layer
+   * - 3
+     - CNN
+     - Clear K NFI science image, bkg-sub & resolved into B & uncertainty layer
+   * - 3
+     - VAM
+     - Mosaic derived wind velocity maps extracted from MP’s: 1440 pos. angles at various altitudes
+   * - 3
+     - VAN
+     - NFI derived wind velocity maps extracted from MP’s: 1440 pos. angles at various altitudes
+   * - 3
+     - PAM
+     - Polarized low-noise science mosaic, bkg-sub & resolved into B, pB, & uncertainty layer
+
+
+Calibration Product Codes
+-------------------------
+
+.. list-table::
+   :header-rows: 1
+
+   * - Level
+     - Code
+     - Description
+   * - 0
+     - DK1, DK2, DK3, DK4, DY1, DY2, DY3, DY4
+     - Calibration Images: polarizer in dark pos.; stim lamp off (DK) or on (DY)
+   * - 0
+     - OV1, OV2, OV3, OV4
+     - Calibration Image: CCD over-scan
+   * - 0
+     - XI1, XI2, XI3, XI4
+     - Experimental image (no set parameters; variable crop)
+   * - 1
+     - BD1, BD2, BD3, BD4
+     - Calibration: Deficient Pixel (Boolean) Map
+   * - 1
+     - FQ1, FQ2, FQ3, FQ4
+     - Calibration: Flat-field parameters (quartic polynomial coefficients), by pixel
+   * - 1
+     - GM1, GM2, GM3, GM4, GZ1, GZ2, GZ3, GZ4, GP1, GP2, GP3, GP4, GR1, GR2, GR3, GR4
+     - Calibration: Vignetting functions for the standard polarization (GM, GZ, GP) and clear (GR) states
+   * - 1
+     - SM1, SM2, SM3, SM4, SZ1, SZ2, SZ3, SZ4, SP1, SP2, SP3, SP4, SR1, SR2, SR3, SR4
+     - Calibration: Instrumental additive stray light model for the standard polarization (SM, SZ, SP) and clear (SR) states
+   * - 1
+     - RM1, RM2, RM3, RM4, RZ1, RZ2, RZ3, RZ4, RP1, RP2, RP3, RP4, RC1, RC2, RC3, RC4
+     - Calibration: Point Spread Function model for the standard polarization and clear states
+   * - Q
+     - CFN
+     - QuickPUNCH NFI images F corona model
+   * - Q
+     - CFM
+     - QuickPUNCH Mosaic images (5.4–80 Rsun) F corona model
+   * - 3
+     - PFM
+     - Polarized mosaic F corona model, resolved into MZP pol. triplets, and uncertainty layer (from MP’s)
+   * - 3
+     - CFM
+     - Clear mosaic F corona model, resolved into image and uncertainty layer (from MC’s)
+   * - 3
+     - PFN
+     - Polarized NFI F-corona model, resolved into MZP pol. triplets, and uncertainty layer (from NP’s)
+   * - 3
+     - CFN
+     - Clear NFI F-corona model, resolved into image and uncertainty layer (from NC’s)
+   * - 3
+     - PSM
+     - Polarized mosaic stellar model, resolved into MZP pol. triplets, and uncertainty layer (from MP’s)
+   * - 3
+     - CSM
+     - Clear mosaic stellar model, resolved into image and uncertainty layer (from MC’s)
+   * - 3
+     - PSN
+     - Polarized NFI stellar model, resolved into MZP pol. triplets, and uncertainty layer (from MP’s)
+   * - 3
+     - CSN
+     - Clear NFI stellar model, resolved into image and uncertainty layer (from MC’s)