[ENH] Allow the T1-Linear pipeline to go over (subject, session) coup…

…les that do not have data (#1285)

* First pass on issue

* Make it work

* Add todos for more complex refactoring

* Testing wip

* Proposition 2

* Touch clinica_list_of_files_reader

* Change to warning

* Changes upon suggestions

* add todos

* Changes upon suggestions

* WIP generalization

* WIP 2

* Fix clinica_list_of_files_reader

* Generalization/Replacement for clinica_file_reader

* Add forgotten condition

* Changes upon suggestions #1

clinica/pipelines/anatomical/freesurfer/atlas/pipeline.py

@@ -84,10 +84,16 @@ def get_to_process_with_atlases(
                     - set(t1_freesurfer_longitudinal_output_atlas)
                 )
                 t1_freesurfer_output, _ = clinica_file_reader(
-                    subjects, sessions, caps_directory, T1_FS_DESTRIEUX, False
+                    subjects,
+                    sessions,
+                    caps_directory,
+                    T1_FS_DESTRIEUX,
                 )
                 t1_freesurfer_files, _ = clinica_file_reader(
-                    subjects, sessions, caps_directory, atlas_info, False
+                    subjects,
+                    sessions,
+                    caps_directory,
+                    atlas_info,
                 )
                 image_ids = extract_image_ids(t1_freesurfer_files)
                 image_ids_2 = extract_image_ids(t1_freesurfer_output)

clinica/pipelines/anatomical/freesurfer/longitudinal/correction/pipeline.py

@@ -60,7 +60,10 @@ def _build_input_node(self):
         )
         from clinica.utils.exceptions import ClinicaException
         from clinica.utils.input_files import T1_FS_DESTRIEUX, T1_FS_T_DESTRIEUX
-        from clinica.utils.inputs import clinica_file_reader
+        from clinica.utils.inputs import (
+            clinica_file_reader,
+            format_clinica_file_reader_errors,
+        )
         from clinica.utils.stream import cprint
 
         from .utils import (
@@ -117,26 +120,19 @@ def _build_input_node(self):
                     to_process_ids
                 )
 
-        all_errors = []
-        try:
-            # Check that t1-freesurfer has run on the CAPS directory
-            clinica_file_reader(
-                self.subjects, self.sessions, self.caps_directory, T1_FS_DESTRIEUX
-            )
-        except ClinicaException as e:
-            all_errors.append(e)
-        try:
-            # Check that t1-freesurfer-template has run on the CAPS directory
-            clinica_file_reader(
-                self.subjects, list_long_id, self.caps_directory, T1_FS_T_DESTRIEUX
-            )
-        except ClinicaException as e:
-            all_errors.append(e)
-        if len(all_errors) > 0:
-            error_message = "Clinica faced errors while trying to read files in your CAPS directory.\n"
-            for msg in all_errors:
-                error_message += str(msg)
-            raise ClinicaException(error_message)
+        _, errors_destrieux = clinica_file_reader(
+            self.subjects, self.sessions, self.caps_directory, T1_FS_DESTRIEUX
+        )
+        _, errors_t_destrieux = clinica_file_reader(
+            self.subjects, list_long_id, self.caps_directory, T1_FS_T_DESTRIEUX
+        )
+        all_errors = [errors_destrieux, errors_t_destrieux]
+
+        if any(all_errors):
+            message = "Clinica faced errors while trying to read files in your CAPS directory.\n"
+            for error, info in zip(all_errors, [T1_FS_DESTRIEUX, T1_FS_T_DESTRIEUX]):
+                message += format_clinica_file_reader_errors(error, info)
+            raise ClinicaException(message)
 
         save_part_sess_long_ids_to_tsv(
             self.subjects, self.sessions, list_long_id, self.base_dir / self.name

clinica/pipelines/anatomical/freesurfer/longitudinal/template/pipeline.py

@@ -32,11 +32,7 @@ def get_processed_images(
         image_ids: List[str] = []
         if caps_directory.is_dir():
             t1_freesurfer_files, _ = clinica_file_reader(
-                list_participant_id,
-                list_long_id,
-                caps_directory,
-                T1_FS_T_DESTRIEUX,
-                False,
+                list_participant_id, list_long_id, caps_directory, T1_FS_T_DESTRIEUX
             )
             image_ids = [
                 re.search(r"(sub-[a-zA-Z0-9]+)_(long-[a-zA-Z0-9]+)", file).group()
@@ -95,7 +91,7 @@ def _build_input_node(self):
         from clinica.utils.exceptions import ClinicaCAPSError, ClinicaException
         from clinica.utils.filemanip import extract_subjects_sessions_from_filename
         from clinica.utils.input_files import T1_FS_DESTRIEUX
-        from clinica.utils.inputs import clinica_file_reader
+        from clinica.utils.inputs import clinica_file_filter
         from clinica.utils.longitudinal import (
             get_long_id,
             get_participants_long_id,
@@ -153,16 +149,10 @@ def _build_input_node(self):
                 self.subjects, self.sessions = extract_subjects_sessions_from_filename(
                     to_process_ids
                 )
-        try:
-            clinica_file_reader(
-                self.subjects, self.sessions, self.caps_directory, T1_FS_DESTRIEUX
-            )
-        except ClinicaException as e:
-            err_msg = (
-                "Clinica faced error(s) while trying to read files in your CAPS directory.\n"
-                + str(e)
-            )
-            raise ClinicaCAPSError(err_msg)
+
+        _, self.subjects, self.sessions = clinica_file_filter(
+            self.subjects, self.sessions, self.caps_directory, T1_FS_DESTRIEUX
+        )
 
         long_ids = get_participants_long_id(self.subjects, self.sessions)
         save_part_sess_long_ids_to_tsv(

clinica/pipelines/anatomical/freesurfer/t1/pipeline.py

@@ -27,7 +27,7 @@ def get_processed_images(
         image_ids: List[str] = []
         if caps_directory.is_dir():
             t1_freesurfer_files, _ = clinica_file_reader(
-                subjects, sessions, caps_directory, T1_FS_DESTRIEUX, False
+                subjects, sessions, caps_directory, T1_FS_DESTRIEUX
             )
             image_ids = extract_image_ids(t1_freesurfer_files)
         return image_ids
@@ -97,7 +97,7 @@ def _build_input_node(self):
             save_participants_sessions,
         )
         from clinica.utils.input_files import T1W_NII
-        from clinica.utils.inputs import clinica_file_reader
+        from clinica.utils.inputs import clinica_file_filter
         from clinica.utils.stream import cprint
         from clinica.utils.ux import print_images_to_process
 
@@ -130,17 +130,10 @@ def _build_input_node(self):
                     to_process_ids
                 )
 
-        t1w_files, error_message = clinica_file_reader(
-            self.subjects,
-            self.sessions,
-            self.bids_directory,
-            T1W_NII,
-            raise_exception=False,
+        t1w_files, self.subjects, self.sessions = clinica_file_filter(
+            self.subjects, self.sessions, self.bids_directory, T1W_NII
         )
 
-        if error_message:
-            cprint(error_message, lvl="warning")
-
         if not t1w_files:
             raise ClinicaException("Empty dataset or already processed")
 

clinica/pipelines/dwi/preprocessing/fmap/pipeline.py

@@ -36,7 +36,7 @@ def get_processed_images(
         image_ids: List[str] = []
         if caps_directory.is_dir():
             preproc_files, _ = clinica_file_reader(
-                subjects, sessions, caps_directory, DWI_PREPROC_NII, False
+                subjects, sessions, caps_directory, DWI_PREPROC_NII
             )
             image_ids = extract_image_ids(preproc_files)
         return image_ids

clinica/pipelines/dwi/preprocessing/t1/pipeline.py

@@ -40,7 +40,7 @@ def get_processed_images(
         image_ids: List[str] = []
         if caps_directory.is_dir():
             preproc_files, _ = clinica_file_reader(
-                subjects, sessions, caps_directory, DWI_PREPROC_NII, False
+                subjects, sessions, caps_directory, DWI_PREPROC_NII
             )
             image_ids = extract_image_ids(preproc_files)
         return image_ids

clinica/pipelines/machine_learning_spatial_svm/spatial_svm_pipeline.py

@@ -63,7 +63,11 @@ def _build_input_node(self):
             pet_volume_normalized_suvr_pet,
             t1_volume_final_group_template,
         )
-        from clinica.utils.inputs import clinica_file_reader, clinica_group_reader
+        from clinica.utils.inputs import (
+            clinica_file_reader,
+            clinica_group_reader,
+            format_clinica_file_reader_errors,
+        )
         from clinica.utils.ux import print_groups_in_caps_directory
 
         if not (
@@ -118,15 +122,16 @@ def _build_input_node(self):
                 f"Image type {self.parameters['orig_input_data_ml']} unknown."
             )
 
-        try:
-            input_image, _ = clinica_file_reader(
-                self.subjects,
-                self.sessions,
-                self.caps_directory,
-                caps_files_information,
+        input_image, caps_error = clinica_file_reader(
+            self.subjects,
+            self.sessions,
+            self.caps_directory,
+            caps_files_information,
+        )
+        if caps_error:
+            all_errors.append(
+                format_clinica_file_reader_errors(caps_error, caps_files_information)
             )
-        except ClinicaException as e:
-            all_errors.append(e)
 
         try:
             dartel_input = clinica_group_reader(
@@ -137,7 +142,7 @@ def _build_input_node(self):
             all_errors.append(e)
 
         # Raise all errors if some happened
-        if len(all_errors) > 0:
+        if any(all_errors):
             error_message = "Clinica faced errors while trying to read files in your CAPS directories.\n"
             for msg in all_errors:
                 error_message += str(msg)

clinica/pipelines/pet/linear/pipeline.py

@@ -67,52 +67,50 @@ def _build_input_node(self):
         )
         from clinica.utils.image import get_mni_template
         from clinica.utils.input_files import T1W_NII, T1W_TO_MNI_TRANSFORM
-        from clinica.utils.inputs import clinica_file_reader
+        from clinica.utils.inputs import (
+            clinica_file_reader,
+            format_clinica_file_reader_errors,
+        )
         from clinica.utils.stream import cprint
         from clinica.utils.ux import print_images_to_process
 
         self.ref_template = get_mni_template("t1")
         self.ref_mask = get_suvr_mask(self.parameters["suvr_reference_region"])
 
         # Inputs from BIDS directory
-        try:
-            pet_files, _ = clinica_file_reader(
-                self.subjects,
-                self.sessions,
-                self.bids_directory,
-                self._get_pet_scans_query(),
-            )
-        except ClinicaException as e:
-            err = (
-                "Clinica faced error(s) while trying to read pet files in your BIDS directory.\n"
-                + str(e)
+        pet_files, pet_errors = clinica_file_reader(
+            self.subjects,
+            self.sessions,
+            self.bids_directory,
+            self._get_pet_scans_query(),
+        )
+        if pet_errors:
+            raise ClinicaBIDSError(
+                format_clinica_file_reader_errors(
+                    pet_errors, self._get_pet_scans_query()
+                )
             )
-            raise ClinicaBIDSError(err)
 
         # T1w file:
-        try:
-            t1w_files, _ = clinica_file_reader(
-                self.subjects, self.sessions, self.bids_directory, T1W_NII
-            )
-        except ClinicaException as e:
-            err = (
-                "Clinica faced error(s) while trying to read t1w files in your BIDS directory.\n"
-                + str(e)
+        t1w_files, t1w_errors = clinica_file_reader(
+            self.subjects, self.sessions, self.bids_directory, T1W_NII
+        )
+        if t1w_errors:
+            raise ClinicaBIDSError(
+                format_clinica_file_reader_errors(t1w_errors, T1W_NII)
             )
-            raise ClinicaBIDSError(err)
 
         # Inputs from t1-linear pipeline
         # Transformation files from T1w files to MNI:
-        try:
-            t1w_to_mni_transformation_files, _ = clinica_file_reader(
-                self.subjects, self.sessions, self.caps_directory, T1W_TO_MNI_TRANSFORM
-            )
-        except ClinicaException as e:
-            err = (
-                "Clinica faced error(s) while trying to read transformation files in your CAPS directory.\n"
-                + str(e)
+        t1w_to_mni_transformation_files, t1w_to_mni_errors = clinica_file_reader(
+            self.subjects, self.sessions, self.caps_directory, T1W_TO_MNI_TRANSFORM
+        )
+        if t1w_to_mni_errors:
+            raise ClinicaCAPSError(
+                format_clinica_file_reader_errors(
+                    t1w_to_mni_errors, T1W_TO_MNI_TRANSFORM
+                )
             )
-            raise ClinicaCAPSError(err)
 
         if len(self.subjects):
             print_images_to_process(self.subjects, self.sessions)