Merge branch '3.9.0x' into hla-1385_configuration_file_update_for_ali…

…gnment_11_21_24

CHANGELOG.rst

@@ -18,11 +18,15 @@ number of the code change for that issue.  These PRs can be viewed at:
     https://github.com/spacetelescope/drizzlepac/pulls
 
 
-3.7.2 (unreleased)
+3.8.0 (unreleased)
 ==================
 
 - Update to pipeline and SVM alignment configuration parameters. [1912]
 
+- Include a minimum RMS value for the SBC detector, as is done for the other
+  detectors, as there seems to be a lot of noise in the source catalogs due to 
+  a low detection threshold. [#1908]
+
 - Force an exit with a return code, KEYWORD_UPDATE_PROBLEM, in try/exception block
   when invoking refine_product_headers in hapsequencer.py and hapmultisequencer.py.
   If the FITS header keywords are not properly updated, this can cause errors during
@@ -52,8 +56,6 @@ number of the code change for that issue.  These PRs can be viewed at:
 
 - Addressed additional issures related to numpy 2.0 scalar promotion. [#1875]
 
-- Added new header keywords and match requirements for relative fitting. [#1860]
-
 - Update to HDRTABLE for MVM products to include SVM rootname and SVM creation date. [#1846]
 
 - Added python 3.12 to testing matrix for Jenkins and github actions. [#1843]

drizzlepac/align.py

@@ -986,16 +986,11 @@ def determine_fit_quality(
 
         # Execute checks
         nmatches_check = False
-
-        if item.meta['fit method']!='relative':
-            required_nmatches = align_pars["run_align"]["mosaic_fitgeom_list"][fitgeom]
-        else:
-            required_nmatches = align_pars["run_align"]["mosaic_relgeom_list"][fitgeom]
-
-        # If the number of matches is more than requirement set in configuration (json) files
-        # OR the fit RMS is higher --> fit is compromised below
-        if num_xmatches >= required_nmatches or (num_xmatches >= 2 and fit_rms_val > 0.5):
+        if num_xmatches >= align_pars["run_align"]["mosaic_fitgeom_list"][fitgeom] or (
+            num_xmatches >= 2 and fit_rms_val > 0.5
+        ):
             nmatches_check = True
+
         radial_offset_check = False
         radial_offset = (
             math.sqrt(

drizzlepac/haputils/catalog_utils.py

@@ -289,6 +289,15 @@ def compute_background(self, box_size, win_size,
                 is_zero_background_defined = True
                 log.info(f"Input image contains excessive zero values in the background. Median: {self.bkg_median:.6f} RMS: {self.bkg_rms_median:.6f}")
 
+                # Compute a minimum rms value based upon information directly from the data
+                minimum_rms = 1.0 / self.keyword_dict['texpo_time']
+                if np.nan_to_num(self.bkg_rms_median) < minimum_rms:
+                    self.bkg_rms_median = minimum_rms
+                    log.info("")
+                    log.info(f"Minimum RMS of input based upon the total exposure time: {minimum_rms:.6f}")
+                    log.info(f"Median RMS has been updated - Median: {self.bkg_median:.6f} RMS: {self.bkg_rms_median:.6f}")
+                    log.info("")
+
         # BACKGROUND COMPUTATION 2 (sigma_clipped_stats)
         # If the input data is not the unusual case of SBC "excessive zero background", compute
         # a sigma-clipped background which returns only single values for mean,
@@ -343,17 +352,18 @@ def compute_background(self, box_size, win_size,
                 log.info("")
 
             # Compute a minimum rms value based upon information directly from the data
-            if self.keyword_dict["detector"].upper() != 'SBC':
-                minimum_rms = self.keyword_dict['atodgn'] * self.keyword_dict['readnse'] \
-                              * self.keyword_dict['ndrizim'] / self.keyword_dict['texpo_time']
-
-                # Compare a minimum rms based upon input characteristics versus the one computed and use
-                # the larger of the two values.
-                if (bkg_rms < minimum_rms):
-                    bkg_rms = minimum_rms
-                    log.info(f"Mimimum RMS of input based upon the readnoise, gain, number of exposures, and total exposure time: {minimum_rms:.6f}")
-                    log.info(f"Sigma-clipped RMS has been updated - Background mean: {bkg_mean:.6f}  median: {bkg_median:.6f}  rms: {bkg_rms:.6f}")
-                    log.info("")
+            if self.keyword_dict['detector'].upper() != 'SBC':
+                minimum_rms = np.sqrt(self.keyword_dict['numexp']) * self.keyword_dict['readnse'] / self.keyword_dict['texpo_time']
+            else:
+                minimum_rms = np.sqrt(np.clip(bkg_median * self.keyword_dict['texpo_time'], a_min=1.0, a_max=None)) / self.keyword_dict['texpo_time']
+
+            # Compare a minimum rms based upon input characteristics versus the rms computed and use
+            # the larger of the two values.
+            if (bkg_rms < minimum_rms):
+               bkg_rms = minimum_rms
+               log.info(f"Minimum RMS of input based upon the readnoise, number of exposures, and total exposure time: {minimum_rms:.6f}")
+               log.info(f"Sigma-clipped RMS has been updated - Background mean: {bkg_mean:.6f}  median: {bkg_median:.6f}  rms: {bkg_rms:.6f}")
+               log.info("")
 
             # Generate two-dimensional background and rms images with the attributes of
             # the input data, but the content based on the sigma-clipped statistics.
@@ -490,6 +500,7 @@ def _get_header_data(self):
         keyword_dict["texpo_time"] = self.imghdu[0].header["TEXPTIME"]
         keyword_dict["exptime"] = self.imghdu[0].header["EXPTIME"]
         keyword_dict["ndrizim"] = self.imghdu[0].header["NDRIZIM"]
+        keyword_dict["numexp"] = self.imghdu[0].header["NUMEXP"]
         if keyword_dict["detector"].upper() != "SBC":
             if keyword_dict["instrument"].upper() == 'WFPC2':
                 atodgn = self._get_max_key_value(self.imghdu[0].header, 'ATODGAIN')

pyproject.toml

@@ -48,7 +48,7 @@ dependencies = [
     "lxml",
     "PyPDF2",
     "scikit-image>=0.14.2",
-    "numpy<2.0",
+    "numpy>2.0",
 ]
 dynamic = [
     "version",

requirements-dev.txt

@@ -6,7 +6,7 @@ git+https://github.com/spacetelescope/stsci.tools.git
 git+https://github.com/spacetelescope/stwcs.git
 git+https://github.com/spacetelescope/stregion.git
 #git+https://github.com/astropy/astroquery.git
-numpy<2.0.dev0
+numpy>2.0.dev0
 scipy>=0.0.dev0
 pyerfa>=0.0.dev0
 astropy>=0.0.dev0

tests/hap/test_svm_j97e06.py

@@ -187,7 +187,7 @@ def test_svm_wcs(gather_output_data):
         print("\ntest_svm_wcs.  WCSNAME: {} Output file: {}".format(wcsname, tdp))
         assert WCS_SUB_NAME in wcsname, f"WCSNAME is not as expected for file {tdp}."
 
-
+@pytest.mark.skip(reason="temporarily skipped")
 def test_svm_point_cat_numsources(gather_output_data):
    # Check that the point catalogs have the expected number of sources
     cat_files = [files for files in gather_output_data if files.lower().endswith("point-cat.ecsv")]
@@ -202,7 +202,7 @@ def test_svm_point_cat_numsources(gather_output_data):
     bad_cats = [cat for cat in valid_cats if not valid_cats[cat][0]]
     assert len(bad_cats) == 0,  f"Point Catalog(s) {bad_cats} had {valid_cats} sources, expected {EXPECTED_POINT_SOURCES}"
 
-
+@pytest.mark.skip(reason="temporarily skipped")
 def test_svm_segment_cat_numsources(gather_output_data):
    # Check that the point catalogs have the expected number of sources
     cat_files = [files for files in gather_output_data if files.lower().endswith("segment-cat.ecsv")]