Skip to content

Commit

Permalink
avoid unnecessary clones when getting crds info
Browse files Browse the repository at this point in the history
  • Loading branch information
braingram committed Aug 1, 2024
1 parent fc937a2 commit aef7eb5
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 19 deletions.
8 changes: 1 addition & 7 deletions src/stpipe/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,13 +184,7 @@ def get_config_from_reference(cls, dataset, disable=None, crds_observatory=None)
logger.debug("Retrieving all substep parameters from CRDS")
#
# Iterate over the steps in the pipeline
with cls._datamodels_open(dataset, asn_n_members=1) as model:
if isinstance(model, Sequence):
crds_parameters = model._models[0].get_crds_parameters()
crds_observatory = model.crds_observatory
else:
crds_parameters = model.get_crds_parameters()
crds_observatory = model.crds_observatory
crds_parameters, crds_observatory = cls._get_crds_info(dataset)

for cal_step in cls.step_defs.keys():
cal_step_class = cls.step_defs[cal_step]
Expand Down
57 changes: 45 additions & 12 deletions src/stpipe/step.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
Step
"""

import contextlib
import gc
import os
import sys
Expand Down Expand Up @@ -755,6 +756,42 @@ def search_attr(self, attribute, default=None, parent_first=False):
value = default
return value

@classmethod
def _get_crds_info(cls, init):
"""
Using a datamodel, filename or container get the:
- crds parameters
- crds observatory
without re-opening, cloning or copying the input.
Parameters
----------
init : `stpipe.datamodel.AbstractDataModel` or str
A datamodel, container, or filename. This method uses
duck-typing checking for "crds_parameters" and "crds_observatory"
attributes (similar to those in
`stpipe.datamodel.AbstractDataModel`).
Returns
-------
parameters : dict
Result of get_crds_parameters
observatory : str
Result of crds_observatory
"""
if hasattr(init, "get_crds_parameters") and hasattr(init, "crds_observatory"):
ctx = contextlib.nullcontext(init)
else:
ctx = cls._datamodels_open(init, asn_n_members=1)

Check warning on line 786 in src/stpipe/step.py

View check run for this annotation

Codecov / codecov/patch

src/stpipe/step.py#L786

Added line #L786 was not covered by tests

with ctx as model_or_container:
if isinstance(model_or_container, Sequence):
model = model_or_container[0]

Check warning on line 790 in src/stpipe/step.py

View check run for this annotation

Codecov / codecov/patch

src/stpipe/step.py#L790

Added line #L790 was not covered by tests
else:
model = model_or_container
return model.get_crds_parameters(), model.crds_observatory

def _precache_references(self, input_file):
"""Because Step precaching precedes calls to get_reference_file() almost
immediately, true precaching has been moved to Pipeline where the
Expand Down Expand Up @@ -817,12 +854,12 @@ def get_reference_file(self, input_file, reference_file_type):
else:
return ""
else:
with self.open_model(input_file) as model:
reference_name = crds_client.get_reference_file(
model.get_crds_parameters(),
reference_file_type,
model.crds_observatory,
)
crds_parameters, crds_observatory = self._get_crds_info(input_file)
reference_name = crds_client.get_reference_file(

Check warning on line 858 in src/stpipe/step.py

View check run for this annotation

Codecov / codecov/patch

src/stpipe/step.py#L857-L858

Added lines #L857 - L858 were not covered by tests
crds_parameters,
reference_file_type,
crds_observatory,
)
if reference_name != "N/A":
hdr_name = "crds://" + basename(reference_name)
else:
Expand Down Expand Up @@ -867,15 +904,11 @@ def get_config_from_reference(cls, dataset, disable=None, crds_observatory=None)
if crds_observatory is None:
raise ValueError("Need a valid name for crds_observatory.")
else:

# If the dataset is not an operable instance of AbstractDataModel,
# log as such and return an empty config object
try:
with cls._datamodels_open(dataset, asn_n_members=1) as model:
if isinstance(model, Sequence):
# Pull out first model in ModelContainer
model = model[0]
crds_parameters = model.get_crds_parameters()
crds_observatory = model.crds_observatory
crds_parameters, crds_observatory = cls._get_crds_info(dataset)

Check warning on line 911 in src/stpipe/step.py

View check run for this annotation

Codecov / codecov/patch

src/stpipe/step.py#L911

Added line #L911 was not covered by tests
except (OSError, TypeError, ValueError):
logger.warning("Input dataset is not an instance of AbstractDataModel.")
disable = True
Expand Down

0 comments on commit aef7eb5

Please sign in to comment.