Updates for V4 RFR2 (#102)

* use isoformt for date-obs

* change vignetting paths

* don't include provenance when loading

* make sure all dates are filled

* make sure all dates are filled

* make sure all dates are filled

* decrease number of workers

create_quartic_fit_coeffs.py

@@ -11,11 +11,11 @@
 from punchbowl.level1.quartic_fit import create_constant_quartic_coefficients
 
 # backward
-wfi_vignetting_model_path = "./build_3_review_files/PUNCH_L1_GM1_20240817174727_v2.fits"
-nfi_vignetting_model_path = "./build_3_review_files/PUNCH_L1_GM4_20240819045110_v1.fits"
+wfi_vignetting_model_path = "PUNCH_L1_GM1_20240817174727_v2.fits"
+nfi_vignetting_model_path = "PUNCH_L1_GM4_20240819045110_v1.fits"
 
-wfi_vignette = load_ndcube_from_fits(wfi_vignetting_model_path).data[...]
-nfi_vignette = load_ndcube_from_fits(nfi_vignetting_model_path).data[...]
+wfi_vignette = load_ndcube_from_fits(wfi_vignetting_model_path, include_provenance=False).data[...]
+nfi_vignette = load_ndcube_from_fits(nfi_vignetting_model_path, include_provenance=False).data[...]
 
 wfi_quartic = create_constant_quartic_coefficients((2048, 2048))
 nfi_quartic = create_constant_quartic_coefficients((2048, 2048))
@@ -24,7 +24,11 @@
 nfi_quartic[-2, :, :] = nfi_vignette
 
 meta = NormalizedMetadata.load_template("FQ1", "1")
-meta['DATE-OBS'] = str(datetime.now())
+meta['DATE-OBS'] = datetime.now().isoformat()
+meta['DATE-BEG'] = meta['DATE-OBS'].value
+meta['DATE-END'] = meta['DATE-BEG'].value
+meta['DATE-AVG'] = meta['DATE-BEG'].value
+meta['DATE'] = meta['DATE-END'].value
 
 wfi_cube = NDCube(data=wfi_quartic, meta=meta, wcs=WCS(naxis=3))
 nfi_cube = NDCube(data=nfi_quartic, meta=meta, wcs=WCS(naxis=3))
@@ -33,11 +37,11 @@
 write_ndcube_to_fits(nfi_cube, "nfi_quartic_backward_coeffs.fits")
 
 # forward
-wfi_vignetting_model_path = "./build_3_review_files/PUNCH_L1_GM1_20240817174727_v2.fits"
-nfi_vignetting_model_path = "./build_3_review_files/PUNCH_L1_GM4_20240819045110_v1.fits"
+wfi_vignetting_model_path = "PUNCH_L1_GM1_20240817174727_v2.fits"
+nfi_vignetting_model_path = "PUNCH_L1_GM4_20240819045110_v1.fits"
 
-wfi_vignette = load_ndcube_from_fits(wfi_vignetting_model_path).data[...]
-nfi_vignette = load_ndcube_from_fits(nfi_vignetting_model_path).data[...]
+wfi_vignette = load_ndcube_from_fits(wfi_vignetting_model_path, include_provenance=False).data[...]
+nfi_vignette = load_ndcube_from_fits(nfi_vignetting_model_path, include_provenance=False).data[...]
 
 wfi_quartic = create_constant_quartic_coefficients((2048, 2048))
 nfi_quartic = create_constant_quartic_coefficients((2048, 2048))
@@ -50,7 +54,11 @@
 nfi_quartic[np.isnan(nfi_quartic)] = 0
 
 meta = NormalizedMetadata.load_template("FQ1", "1")
-meta['DATE-OBS'] = str(datetime.now())
+meta['DATE-OBS'] = datetime.now().isoformat()
+meta['DATE-BEG'] = meta['DATE-OBS'].value
+meta['DATE-END'] = meta['DATE-BEG'].value
+meta['DATE-AVG'] = meta['DATE-BEG'].value
+meta['DATE'] = meta['DATE-END'].value
 
 wfi_cube = NDCube(data=wfi_quartic, meta=meta, wcs=WCS(naxis=3))
 nfi_cube = NDCube(data=nfi_quartic, meta=meta, wcs=WCS(naxis=3))

simpunch/level0.py

@@ -126,7 +126,7 @@ def generate_l0_pmzp(input_file: NDCube,
     product_code = input_data.meta["TYPECODE"].value + input_data.meta["OBSCODE"].value
     product_level = "0"
     output_meta = NormalizedMetadata.load_template(product_code, product_level)
-    output_meta["DATE-OBS"] = str(input_data.meta.datetime)
+    output_meta["DATE-OBS"] = input_data.meta.datetime.isoformat()
 
     quartic_coefficients = wfi_quartic_coefficients \
         if input_data.meta["OBSCODE"].value != "4" else nfi_quartic_coefficients
@@ -209,7 +209,7 @@ def generate_l0_cr(input_file: NDCube, path_output: str,
     product_code = input_data.meta["TYPECODE"].value + input_data.meta["OBSCODE"].value
     product_level = "0"
     output_meta = NormalizedMetadata.load_template(product_code, product_level)
-    output_meta["DATE-OBS"] = str(input_data.meta.datetime)
+    output_meta["DATE-OBS"] = input_data.meta.datetime.isoformat()
 
     quartic_coefficients = wfi_quartic_coefficients \
         if input_data.meta["OBSCODE"].value != "4" else nfi_quartic_coefficients

simpunch/level1.py

@@ -347,7 +347,7 @@ def generate_l1_cr(input_file: str, path_output: str, rotation_stage: int, space
 
 
 @flow(log_prints=True, task_runner=DaskTaskRunner(
-    cluster_kwargs={"n_workers": 128, "threads_per_worker": 2},
+    cluster_kwargs={"n_workers": 64, "threads_per_worker": 2},
 ))
 def generate_l1_all(datadir: str, outdir: str) -> None:
     """Generate all level 1 synthetic data.

simpunch/level2.py

@@ -351,7 +351,7 @@ def generate_l2_ctm(input_file: str, path_output: str) -> None:
 
 
 @flow(log_prints=True, task_runner=DaskTaskRunner(
-    cluster_kwargs={"n_workers": 128, "threads_per_worker": 2},
+    cluster_kwargs={"n_workers": 64, "threads_per_worker": 2},
 ))
 def generate_l2_all(datadir: str, outdir: str) -> None:
     """Generate all level 2 synthetic data.

simpunch/level3.py

@@ -215,7 +215,7 @@ def generate_l3_ctm(input_tb: str,
 
 
 @flow(log_prints=True, task_runner=DaskTaskRunner(
-    cluster_kwargs={"n_workers": 128, "threads_per_worker": 2},
+    cluster_kwargs={"n_workers": 64, "threads_per_worker": 2},
 ))
 def generate_l3_all(datadir: str, outdir: str, start_time: datetime, num_repeats: int = 1) -> None:
     """Generate all level 3 synthetic data."""
@@ -251,7 +251,7 @@ def generate_l3_all(datadir: str, outdir: str, start_time: datetime, num_repeats
 
 
 @flow(log_prints=True, task_runner=DaskTaskRunner(
-    cluster_kwargs={"n_workers": 128, "threads_per_worker": 2},
+    cluster_kwargs={"n_workers": 64, "threads_per_worker": 2},
 ))
 def generate_l3_all_fixed(datadir: str, outdir: str, times: list[datetime], file_pb: str, file_tb: str) -> None:
     """Generate level 3 synthetic data at specified times with a fixed GAMERA model."""