Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

small fix: skipping files for qc that raise an exception #45

Merged
merged 1 commit into from
Dec 11, 2024
+77 −67
Merged

small fix: skipping files for qc that raise an exception #45

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 77 additions & 67 deletions src/iblphotometry/qc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,74 +113,84 @@ def run_qc(
qc_results = []
for eid in tqdm(eids):
print(eid)
# get photometry data
raw_dfs = data_loader.load_photometry_data(eid=eid)
signal_bands = list(raw_dfs.keys())
brain_regions = raw_dfs[signal_bands[0]]

# get behavioral data
# TODO this should be provided
sl = SessionLoader(eid=eid, one=data_loader.one)
# for caroline
# trials = sl.load_trials(
# collection='alf/task_00'
# ) # this is necessary fo caroline
trials = sl.load_trials() # should be good for all others

# the old way
# trials = data_loader.one.load_dataset(eid, '*trials.table.pqt')

for band in signal_bands:
raw_tf = raw_dfs[band]
for region in brain_regions:
qc_result = qc_series(
raw_tf[region], qc_metrics['raw'], sliding_kwargs=None, eid=eid
)
qc_results.append(
dict(eid=eid, pipeline='raw', band=band, region=region, **qc_result)
)

# run the pipelines and qc on the processed data
# here it needs to be specified if one band is a reference of the other
for pipeline_name, pipeline in pipelines_reg.items():
if 'reference' in sigref_mapping: # this is for isosbestic pipelines
proc_tf = run_pipeline(
pipeline,
raw_dfs[sigref_mapping['signal']],
raw_dfs[sigref_mapping['reference']],
)
else:
# FIXME this fails for true-multiband
# this hack works for single-band
# possible fix could be that signal could be a list
proc_tf = run_pipeline(pipeline, raw_dfs[sigref_mapping['signal']])

for region in brain_regions:
# sliding qc of the processed data
qc_proc = qc_series(
proc_tf[region],
qc_metrics=qc_metrics['processed'],
sliding_kwargs=qc_metrics['sliding_kwargs'],
eid=eid,
brain_region=region,
)

# qc with metrics that use behavior
qc_resp = qc_series(
proc_tf[region],
qc_metrics['response'],
trials=trials,
eid=eid,
brain_region=region,
)
qc_result = qc_proc | qc_resp
qc_results.append(
dict(
try:
# get photometry data
raw_dfs = data_loader.load_photometry_data(eid=eid)
signal_bands = list(raw_dfs.keys())
brain_regions = raw_dfs[signal_bands[0]]

# get behavioral data
# TODO this should be provided
sl = SessionLoader(eid=eid, one=data_loader.one)
# for caroline
# trials = sl.load_trials(
# collection='alf/task_00'
# ) # this is necessary fo caroline
trials = sl.load_trials() # should be good for all others

# the old way
# trials = data_loader.one.load_dataset(eid, '*trials.table.pqt')

for band in signal_bands:
raw_tf = raw_dfs[band]
for region in brain_regions:
qc_result = qc_series(
raw_tf[region], qc_metrics['raw'], sliding_kwargs=None, eid=eid
)
qc_results.append(
dict(
eid=eid,
pipeline='raw',
band=band,
region=region,
**qc_result,
)
)

# run the pipelines and qc on the processed data
# here it needs to be specified if one band is a reference of the other
for pipeline_name, pipeline in pipelines_reg.items():
if 'reference' in sigref_mapping: # this is for isosbestic pipelines
proc_tf = run_pipeline(
pipeline,
raw_dfs[sigref_mapping['signal']],
raw_dfs[sigref_mapping['reference']],
)
else:
# FIXME this fails for true-multiband
# this hack works for single-band
# possible fix could be that signal could be a list
proc_tf = run_pipeline(pipeline, raw_dfs[sigref_mapping['signal']])

for region in brain_regions:
# sliding qc of the processed data
qc_proc = qc_series(
proc_tf[region],
qc_metrics=qc_metrics['processed'],
sliding_kwargs=qc_metrics['sliding_kwargs'],
eid=eid,
pipeline=pipeline_name,
region=region,
**qc_result,
brain_region=region,
)
)

# qc with metrics that use behavior
qc_resp = qc_series(
proc_tf[region],
qc_metrics['response'],
trials=trials,
eid=eid,
brain_region=region,
)
qc_result = qc_proc | qc_resp
qc_results.append(
dict(
eid=eid,
pipeline=pipeline_name,
region=region,
**qc_result,
)
)
except Exception as e:
logger.warning(f'{eid}: failure: {type(e).__name__}:{e}')

gc.collect()
return qc_results
Loading