diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..f05d226 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,23 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true + +# Matches multiple files with brace expansion notation +# Set default charset +[*.{js,py}] +charset = utf-8 + +# 4 space indentation +[*.py] +indent_style = space +indent_size = 4 + +# Tab indentation (no size specified) +[Makefile] +indent_style = tab diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml new file mode 100644 index 0000000..c7d8ae6 --- /dev/null +++ b/.github/workflows/pylint.yml @@ -0,0 +1,23 @@ +name: Pylint + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.6"] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint + - name: Analysing the code with pylint + run: | + pylint rec_to_binaries/ diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml new file mode 100644 index 0000000..78a0069 --- /dev/null +++ b/.github/workflows/unit_test.yml @@ -0,0 +1,25 @@ +name: Project Tests + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + strategy: + matrix: + python-version: ["3.6"] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: conda-incubator/setup-miniconda@v2 + with: + miniconda-version: "latest" + activate-environment: rec_to_binaries + environment-file: environment.yml + - run: pytest -vv diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..5a065c2 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,44 @@ +#options - https://github.com/Qiskit/openqasm/blob/master/.pylintrc +# https://pylint.pycqa.org/en/latest/technical_reference/features.html + + +[MESSAGES CONTROL] +disable= + missing-docstring, + too-few-public-methods, + too-many-instance-attributes, + line-too-long, + too-many-arguments, + logging-fstring-interpolation, + consider-using-f-string, # consider removing in the future + import-error, # consider removing after finding a better solution + + +[TYPECHECK] +ignored-modules = numpy, pandas, scipy.stats + + +[BASIC] +# Good variable names which should always be accepted, separated by a comma +good-names=_, id + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Regular expression matching correct attribute names +attr-rgx=[A-Za-z_][A-Za-z0-9_]{0,30}$ + +# Naming hint for attribute names +attr-name-hint=[A-Za-z_][A-Za-z0-9_]{0,30}$ + +# Regular expression matching correct argument names +argument-rgx=[A-Za-z_][A-Za-z0-9_]{0,30}$ + +# Naming hint for argument names +argument-name-hint=[A-Za-z_][A-Za-z0-9_]{0,30}$ + +# Regular expression matching correct variable names +variable-rgx=[A-Za-z_][A-Za-z0-9_]{0,30}$ + +# Naming hint for variable names +variable-name-hint=[A-Za-z_][A-Za-z0-9_]{0,30}$ diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..e46c1a4 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "python.linting.pylintEnabled": true, + "python.linting.enabled": true, + "python.linting.flake8Enabled": false +} \ No newline at end of file diff --git a/environment.yml b/environment.yml index ff724a9..ebfd949 100644 --- a/environment.yml +++ b/environment.yml @@ -14,6 +14,7 @@ dependencies: - pytest - pytest-cov - coveralls +- Faker - pip - pip: - mountainlab-pytools diff --git a/rec_to_binaries/adjust_timestamps.py b/rec_to_binaries/adjust_timestamps.py index 4c13c09..b44597d 100644 --- a/rec_to_binaries/adjust_timestamps.py +++ b/rec_to_binaries/adjust_timestamps.py @@ -10,10 +10,10 @@ import numpy as np import pandas as pd +from scipy.stats import linregress from rec_to_binaries.create_system_time import infer_systime from rec_to_binaries.read_binaries import (readTrodesExtractedDataFile, write_trodes_extracted_datafile) -from scipy.stats import linregress logger = getLogger(__name__) @@ -53,20 +53,20 @@ def _regress_timestamps(trodestime, systime): Unix time """ - NANOSECONDS_TO_SECONDS = 1E9 + nanoseconds_to_seconds = 1E9 # Convert systime_seconds = np.asarray(systime).astype( - np.float64) / NANOSECONDS_TO_SECONDS + np.float64) / nanoseconds_to_seconds trodestime_index = np.asarray(trodestime).astype(np.float64) - slope, intercept, r_value, p_value, std_err = linregress( + slope, intercept = linregress( trodestime_index, systime_seconds) adjusted_timestamps = intercept + slope * trodestime_index - return (adjusted_timestamps * NANOSECONDS_TO_SECONDS).astype(np.int64) + return (adjusted_timestamps * nanoseconds_to_seconds).astype(np.int64) -def _insert_new_data(data_file, df): +def _insert_new_data(data_file, data_frame): """ Replaces the `data` in the extracted data file with a new one. @@ -74,7 +74,7 @@ def _insert_new_data(data_file, df): ---------- data_file : dict Original data file as read in by `readTrodesExtractedDataFile` - df : pandas.DataFrame + data_frame : pandas.DataFrame New data Returns @@ -84,7 +84,7 @@ def _insert_new_data(data_file, df): """ new_data_file = data_file.copy() - new_data_file['data'] = np.asarray(df.to_records(index=False)) + new_data_file['data'] = np.asarray(data_frame.to_records(index=False)) new_data_file['fields'] = ''.join( [f'<{name} {dtype}>' for name, (dtype, _) in new_data_file['data'].dtype.fields.items()]) @@ -113,7 +113,7 @@ def fix_timestamp_lag(continuoustime_filename): data_file = readTrodesExtractedDataFile(continuoustime_filename) if 'systime' not in data_file['data'].dtype.names: - logger.warn("No `systime`. Inferring from `system_time_at_creation` timestamp" + logger.warning("No `systime`. Inferring from `system_time_at_creation` timestamp" " as a function of the `clockrate` and `trodestime`") new_data = infer_systime(data_file) else: diff --git a/rec_to_binaries/core.py b/rec_to_binaries/core.py index 58f2dee..6b72c6c 100644 --- a/rec_to_binaries/core.py +++ b/rec_to_binaries/core.py @@ -34,6 +34,7 @@ def extract_trodes_rec_file(data_dir, parallel_instances=1, use_day_config=True, trodes_version=None): + # pylint: disable=too-many-branches, too-many-locals, too-many-statements """Extracting Trodes rec files. Following the Frank Lab directory structure for raw ephys data, will @@ -205,11 +206,11 @@ def extract_trodes_rec_file(data_dir, use_day_config=use_day_config) if adjust_timestamps_for_mcu_lag: - ''''There is some jitter in the arrival times of packets from the MCU (as - reflected in the sysclock records in the .rec file. If we assume that - the Trodes clock is actually regular, and that any episodes of lag are - fairly sporadic, we can recover the correspondence between trodestime - and system (wall) time.''' + # There is some jitter in the arrival times of packets from the MCU (as + # reflected in the sysclock records in the .rec file. If we assume that + # the Trodes clock is actually regular, and that any episodes of lag are + # fairly sporadic, we can recover the correspondence between trodestime + # and system (wall) time. preprocessing_dir = animal_info.get_preprocessing_dir() filenames = glob.glob(os.path.join( preprocessing_dir, '**', '*.continuoustime.dat'), recursive=True) @@ -243,9 +244,7 @@ def convert_binaries_to_hdf5(data_dir, animal, out_dir=None, dates=None, convert_lfp=True, convert_pos=True, convert_spike=True): - animal_info = td.TrodesAnimalInfo( - data_dir, animal, out_dir=out_dir, dates=dates) - """Converting preprocessed binaries into HDF5 files. + '''Converting preprocessed binaries into HDF5 files. Assume that preprocessing has already been completed using (for example) extract_trodes_rec_file. @@ -266,7 +265,9 @@ def convert_binaries_to_hdf5(data_dir, animal, out_dir=None, dates=None, convert_lfps : bool, optional convert_dio : bool, optional convert_mda : bool, optional - """ + ''' + animal_info = td.TrodesAnimalInfo( + data_dir, animal, out_dir=out_dir, dates=dates) importer = td.TrodesPreprocessingToAnalysis(animal_info) diff --git a/rec_to_binaries/create_system_time.py b/rec_to_binaries/create_system_time.py index 2bebf44..102fc78 100644 --- a/rec_to_binaries/create_system_time.py +++ b/rec_to_binaries/create_system_time.py @@ -41,14 +41,14 @@ def create_systime(clockrate, data, system_time_at_creation): millisecond """ - NANOSECONDS_TO_SECONDS = 1e9 + nanoseconds_to_seconds = 1e9 clockrate = int(clockrate) n_time = data.shape[0] system_time_at_creation = pd.to_datetime( int(system_time_at_creation), unit='ms').value end = (system_time_at_creation + - int((n_time - 1) * NANOSECONDS_TO_SECONDS / clockrate)) + int((n_time - 1) * nanoseconds_to_seconds / clockrate)) systime = pd.date_range( start=system_time_at_creation, diff --git a/rec_to_binaries/read_binaries.py b/rec_to_binaries/read_binaries.py index 4207b9b..9550394 100644 --- a/rec_to_binaries/read_binaries.py +++ b/rec_to_binaries/read_binaries.py @@ -4,7 +4,7 @@ import numpy as np -def readTrodesExtractedDataFile(filename): +def readTrodesExtractedDataFile(filename): # pylint: disable=invalid-name '''Read extracted trodes binary. Parameters @@ -20,24 +20,24 @@ def readTrodesExtractedDataFile(filename): # Check if first line is start of settings block if file.readline().decode().strip() != '': raise Exception("Settings format not supported") - fieldsText = dict() + fields_text = {} for line in file: # Read through block of settings line = line.decode().strip() # filling in fields dict if line != '': settings_name, setting = line.split(': ') - fieldsText[settings_name.lower()] = setting + fields_text[settings_name.lower()] = setting # End of settings block, signal end of fields else: break # Reads rest of file at once, using dtype format generated by parse_dtype() try: - fieldsText['data'] = np.fromfile( - file, dtype=parse_dtype(fieldsText['fields'])) + fields_text['data'] = np.fromfile( + file, dtype=parse_dtype(fields_text['fields'])) except KeyError: - fieldsText['data'] = np.fromfile(file) - return fieldsText + fields_text['data'] = np.fromfile(file) + return fields_text def parse_dtype(fieldstr): @@ -46,7 +46,7 @@ def parse_dtype(fieldstr): Returns: np.dtype ''' # Returns np.dtype from field string - sep = re.split('\s', re.sub(r"\>\<|\>|\<", ' ', fieldstr).strip()) + sep = re.split('\s', re.sub(r"\>\<|\>|\<", ' ', fieldstr).strip()) # pylint: disable=anomalous-backslash-in-string typearr = [] # Every two elemets is fieldname followed by datatype @@ -56,7 +56,7 @@ def parse_dtype(fieldstr): ftype = 'uint32' # Finds if a * is included in datatype if '*' in sep[i + 1]: - temptypes = re.split('\*', sep[i + 1]) + temptypes = re.split('\*', sep[i + 1]) # pylint: disable=anomalous-backslash-in-string # Results in the correct assignment, whether str is num*dtype or dtype*num ftype = temptypes[temptypes[0].isdigit()] repeats = int(temptypes[temptypes[1].isdigit()]) diff --git a/rec_to_binaries/trodes_data.py b/rec_to_binaries/trodes_data.py index 50c8878..7836bf4 100644 --- a/rec_to_binaries/trodes_data.py +++ b/rec_to_binaries/trodes_data.py @@ -1,3 +1,4 @@ +# pylint: disable=too-many-lines import copy import functools import itertools @@ -37,7 +38,7 @@ class TrodesAnimalInfoError(RuntimeError): class TrodesRawFileNameParser: trodes_filename_re = re.compile( - '^(\d*)_([a-zA-Z0-9]*)_(\d*)_{0,1}(\w*)\.{0,1}(.*)\.([a-zA-Z0-9]*)$') + '^(\d*)_([a-zA-Z0-9]*)_(\d*)_{0,1}(\w*)\.{0,1}(.*)\.([a-zA-Z0-9]*)$') # pylint: disable=anomalous-backslash-in-string def __init__(self, filename_str): self.filename = filename_str @@ -51,26 +52,23 @@ def __init__(self, filename_str): self.date) except ValueError: - raise TrodesDataFormatError('Filename ({}) date field is not a true date.'. - format(filename_str)) + raise TrodesDataFormatError(f'Filename ({filename_str}) date field is not a true date.') from None self.name_str = filename_groups[1] self.epochlist_str = filename_groups[2] - epochlist = re.findall('\d\d', self.epochlist_str) + epochlist = re.findall('\d\d', self.epochlist_str) # pylint: disable=anomalous-backslash-in-string if len(epochlist) == 0: - raise TrodesDataFormatError(('Filename ({}) does not match basic trodes file format, epoch list ' - 'could not be parsed.').format(filename_str)) - self.epochtuple = tuple([int(epoch_str) - for epoch_str in epochlist]) + raise TrodesDataFormatError((f'Filename ({filename_str}) does not match basic trodes file format, epoch list ' + 'could not be parsed.')) + self.epochtuple = tuple([int(epoch_str) for epoch_str in epochlist]) # pylint: disable=consider-using-generator self.label = filename_groups[3] self.label_ext = filename_groups[4] self.ext = filename_groups[5] self.filename_no_ext = re.match( - '^(.*)\.(.*)$', self.filename).groups()[0] + '^(.*)\.(.*)$', self.filename).groups()[0] # pylint: disable=anomalous-backslash-in-string else: - raise TrodesDataFormatError('Filename ({}) does not match basic trodes file format.'. - format(filename_str)) + raise TrodesDataFormatError(f'Filename ({filename_str}) does not match basic trodes file format.') @staticmethod def expand_date_str(date_str): @@ -87,15 +85,14 @@ def __init__(self, filename_str): self.time_label = None else: self.export_logfile = False - label_match = re.match('^.*_nt(\d*)\.{0,1}(.*)$', self.label_ext) + label_match = re.match('^.*_nt(\d*)\.{0,1}(.*)$', self.label_ext) # pylint: disable=anomalous-backslash-in-string if label_match is not None and self.ext == 'dat': label_match_group = label_match.groups() self.ntrode = int(label_match_group[0]) self.time_label = label_match_group[1] else: - raise TrodesDataFormatError('Filename ({}) does not match ({}) format.'. - format(filename_str, self.__class__.__name__)) + raise TrodesDataFormatError('Filename ({filename_str}) does not match ({self.__class__.__name__}) format.') class TrodesLFPExtractedFileNameParser(TrodesRawFileNameParser): @@ -112,7 +109,7 @@ def __init__(self, filename_str): else: self.export_logfile = False timestamp_label_match = re.match( - '^timestamps\.{0,1}(.*)$', self.label_ext) + '^timestamps\.{0,1}(.*)$', self.label_ext) # pylint: disable=anomalous-backslash-in-string if timestamp_label_match is not None and self.ext == 'dat': self.timestamp_file = True self.time_label = timestamp_label_match.groups()[0] @@ -121,15 +118,14 @@ def __init__(self, filename_str): else: self.timestamp_file = False self.time_label = None - label_match = re.match('^.*_nt(\d*)ch(\d*)$', self.label_ext) + label_match = re.match('^.*_nt(\d*)ch(\d*)$', self.label_ext) # pylint: disable=anomalous-backslash-in-string if label_match is not None and self.ext == 'dat': label_match_groups = label_match.groups() self.ntrode = int(label_match_groups[0]) self.channel = int(label_match_groups[1]) else: - raise TrodesDataFormatError('Filename ({}) does not match ({}) format.'. - format(filename_str, self.__class__.__name__)) + raise TrodesDataFormatError(f'Filename ({filename_str}) does not match ({self.__class__.__name__}) format.') class TrodesMdaExtractedFileNameParser(TrodesRawFileNameParser): @@ -144,7 +140,7 @@ def __init__(self, filename_str): else: self.export_logfile = False timestamp_label_match = re.match( - '^timestamps\.{0,1}(.*)$', self.label_ext) + '^timestamps\.{0,1}(.*)$', self.label_ext) # pylint: disable=anomalous-backslash-in-string if timestamp_label_match is not None and self.ext == 'mda': self.timestamp_file = True self.time_label = timestamp_label_match.groups()[0] @@ -152,14 +148,13 @@ def __init__(self, filename_str): else: self.timestamp_file = False self.time_label = None - label_match = re.match('^nt(\d*)$', self.label_ext) + label_match = re.match('^nt(\d*)$', self.label_ext) # pylint: disable=anomalous-backslash-in-string if label_match is not None and self.ext == 'mda': label_match_groups = label_match.groups() self.ntrode = int(label_match_groups[0]) else: - raise TrodesDataFormatError('Filename ({}) does not match ({}) format.'. - format(filename_str, self.__class__.__name__)) + raise TrodesDataFormatError(f'Filename ({filename_str}) does not match ({self.__class__.__name__}) format.') class TrodesDIOExtractedFileNameParser(TrodesRawFileNameParser): @@ -178,7 +173,7 @@ def __init__(self, filename_str): if labels[0] == 'dio' and self.ext == 'dat': label_match_groups = re.match( - '([A-Za-z]*)(\d*)\.{0,1}(.*)$', labels[-1]).groups() + '([A-Za-z]*)(\d*)\.{0,1}(.*)$', labels[-1]).groups() # pylint: disable=anomalous-backslash-in-string # string in/out self.direction = label_match_groups[0][1:].lower() # Din/Dout channel @@ -186,19 +181,18 @@ def __init__(self, filename_str): self.time_label = label_match_groups[2] else: - raise TrodesDataFormatError('Filename ({}) does not match ({}) format.'. - format(filename_str, self.__class__.__name__)) + raise TrodesDataFormatError(f'Filename ({filename_str}) does not match ({self.__class__.__name__}) format.') class TrodesPosExtractedFileNameParser(TrodesRawFileNameParser): def __init__(self, filename_str): super().__init__(filename_str) - label_ext_match = re.match('^(\d*)\.?(.*)$', self.label_ext) + label_ext_match = re.match('^(\d*)\.?(.*)$', self.label_ext) # pylint: disable=anomalous-backslash-in-string self.label_ext1 = label_ext_match.groups()[0] self.label_ext2 = label_ext_match.groups()[1] timestamp_label_match = re.match( - '^pos_timestamps\.{0,1}(.*)$', self.label_ext2) + '^pos_timestamps\.{0,1}(.*)$', self.label_ext2) # pylint: disable=anomalous-backslash-in-string if timestamp_label_match is not None and self.ext == 'dat': self.timestamp_file = True self.time_label = timestamp_label_match.groups()[0] @@ -211,15 +205,14 @@ def __init__(self, filename_str): self.pos_label = label_match.groups()[0] else: - raise TrodesDataFormatError('Filename ({}) does not match ({}) format.'. - format(filename_str, self.__class__.__name__)) - + raise TrodesDataFormatError(f'Filename ({filename_str}) does not match ({self.__class__.__name__}) format.') class TrodesAnimalInfo: def __init__(self, base_dir, anim_name, RawFileParser=TrodesRawFileNameParser, out_dir=None, dates=None, trodes_version=None): - self.RawFileNameParser = RawFileParser + # pylint: disable=too-many-locals, too-many-branches, too-many-statements + self.Raw_file_name_parser = RawFileParser self.base_dir = base_dir self.anim_name = anim_name @@ -249,17 +242,16 @@ def __init__(self, base_dir, anim_name, RawFileParser=TrodesRawFileNameParser, continue self.raw_rec_files[date] = {} day_rec_filenames = self._get_rec_paths( - day_path, self.RawFileNameParser) + day_path, self.Raw_file_name_parser) if trodes_version is None: self.trodes_version = self._get_trodes_version(day_path) else: self.trodes_version = trodes_version for rec_filename_parsed, rec_path in day_rec_filenames: if rec_filename_parsed.date != date: - logger.warn(('For rec file ({}) the date field does not match ' - 'the folder date ({}). This should be fixed or could have' - 'unintended parsing consequences.'). - format(rec_filename_parsed.filename, date)) + logger.warning((f'For rec file ({rec_filename_parsed.filename}) the date field does not match ' + f'the folder date ({date}). This should be fixed or could have' + 'unintended parsing consequences.')) try: self.raw_rec_files[date][rec_filename_parsed.epochtuple].append( (rec_filename_parsed, rec_path)) @@ -269,7 +261,7 @@ def __init__(self, base_dir, anim_name, RawFileParser=TrodesRawFileNameParser, self.raw_pos_files[date] = {} day_pos_filenames = self._get_video_tracking_paths( - day_path, self.RawFileNameParser) + day_path, self.Raw_file_name_parser) for pos_filename_parsed, pos_path in day_pos_filenames: raw_pos_file_date_epoch = self.raw_pos_files[date].setdefault( pos_filename_parsed.epochtuple, {}) @@ -278,7 +270,7 @@ def __init__(self, base_dir, anim_name, RawFileParser=TrodesRawFileNameParser, self.raw_h264_files[date] = {} day_h264_filenames = self._get_h264_paths( - day_path, self.RawFileNameParser) + day_path, self.Raw_file_name_parser) for h264_filename_parsed, h264_path in day_h264_filenames: raw_h264_file_date_epoch = self.raw_h264_files[date].setdefault( h264_filename_parsed.epochtuple, {}) @@ -287,7 +279,7 @@ def __init__(self, base_dir, anim_name, RawFileParser=TrodesRawFileNameParser, self.raw_postime_files[date] = {} day_postime_filenames = self._get_video_timestamp_paths( - day_path, self.RawFileNameParser) + day_path, self.Raw_file_name_parser) for postime_filename_parsed, postime_path in day_postime_filenames: raw_postime_file_date_epoch = self.raw_postime_files[date]. \ setdefault(postime_filename_parsed.epochtuple, {}) @@ -296,16 +288,16 @@ def __init__(self, base_dir, anim_name, RawFileParser=TrodesRawFileNameParser, self.raw_poshwframecount_files[date] = {} day_poshwframecount_filenames = self._get_video_hwframecount_paths( - day_path, self.RawFileNameParser) + day_path, self.Raw_file_name_parser) for poshwframecount_filename_parsed, poshwframecount_path in day_poshwframecount_filenames: raw_poshwframecount_file_date_epoch = self.raw_poshwframecount_files[date]. \ - setdefault(poshwframecount_filename_parsed.epochtuple, {}) + setdefault(poshwframecount_filename_parsed.epochtuple, {}) # pylint: disable=invalid-name raw_poshwframecount_file_date_epoch[poshwframecount_filename_parsed.label_ext] = \ (poshwframecount_filename_parsed, poshwframecount_path) self.raw_trodescomments_files[date] = {} day_trodescomments_filenames = self._get_trodes_comments_paths( - day_path, self.RawFileNameParser) + day_path, self.Raw_file_name_parser) for trodescomments_filename_parsed, trodescomments_path in day_trodescomments_filenames: self.raw_trodescomments_files[date][trodescomments_filename_parsed.epochtuple] = \ (trodescomments_filename_parsed, trodescomments_path) @@ -379,8 +371,7 @@ def __repr__(self): f"base_dir={self.__dict__['base_dir']})") @staticmethod - def _get_extracted_datatype_paths_df(directory_entries_df, parser_datatype_fields, sort_on_fields, - ExtractedFileParser): + def _get_extracted_datatype_paths_df(directory_entries_df, parser_datatype_fields, sort_on_fields, ExtractedFileParser): # pylint: disable=invalid-name """ Args: @@ -450,11 +441,10 @@ def _lookup_date_epoch_dict(nested_dict, date, epoch): if return_val is None: return_val = epoch_val else: - raise TrodesAnimalInfoError(('date ({}) and epoch ({}) index ' + raise TrodesAnimalInfoError((f'date ({date}) and epoch ({epoch}) index ' 'returns more than one possible value. ' 'Likely the file structure for ' - 'is invalid and should be fixed.') - .format(date, epoch)) + 'is invalid and should be fixed.')) return return_val @@ -472,8 +462,7 @@ def get_raw_h264_paths(self, date, epoch): self.raw_h264_files, date, epoch) if h264_paths is None: raise KeyError(('Raw h264 video file does not exist ' - 'for animal ({}), date ({}) and epoch ({}).'). - format(self.anim_name, date, epoch)) + 'for animal ({self.anim_name}), date ({date}) and epoch ({epoch}).')) return h264_paths def get_raw_h264_path(self, date, epoch, label_ext): @@ -484,8 +473,7 @@ def get_raw_pos_paths(self, date, epoch): self.raw_pos_files, date, epoch) if pos_paths is None: raise KeyError(('Raw/online position tracking file does not exist ' - 'for animal ({}), date ({}) and epoch ({}).'). - format(self.anim_name, date, epoch)) + 'for animal ({self.anim_name}), date ({date}) and epoch ({epoch}).')) return pos_paths @@ -496,8 +484,7 @@ def get_raw_postime_paths(self, date, epoch): postime_paths = TrodesAnimalInfo._lookup_date_epoch_dict( self.raw_postime_files, date, epoch) if postime_paths is None: - raise KeyError('Online position timestamps file does not exist for animal ({}), date ({}) and epoch ({}).'. - format(self.anim_name, date, epoch)) + raise KeyError(f'Online position timestamps file does not exist for animal ({self.anim_name}), date ({date}) and epoch ({epoch}).') return postime_paths @@ -509,8 +496,7 @@ def get_raw_poshwframecount_paths(self, date, epoch): self.raw_poshwframecount_files, date, epoch) if poshwframecount_paths is None: raise KeyError(('Online position hwFrameCount file does not exist for ' - 'animal ({}), date ({}) and epoch ({}).'). - format(self.anim_name, date, epoch)) + f'animal ({self.anim_name}), date ({date}) and epoch ({epoch}).')) return poshwframecount_paths def get_raw_poshwframecount_path(self, date, epoch, label_ext): @@ -520,9 +506,8 @@ def get_raw_rec_path(self, date, epoch): rec_path = TrodesAnimalInfo._lookup_date_epoch_dict( self.raw_rec_files, date, epoch) if rec_path is None: - raise KeyError(('Rec files does not exist for ' - 'animal ({}), date ({}) and epoch ({}).'). - format(self.anim_name, date, epoch)) + raise KeyError('Rec files does not exist for ' + f'animal ({self.anim_name}), date ({date}) and epoch ({epoch}).') return rec_path def get_raw_dir(self): @@ -538,11 +523,9 @@ def get_preprocessing_date_dir(self, date, stop_error=True): path = os.path.join(self.get_preprocessing_dir(), date) if not os.path.isdir(path) and stop_error: if os.path.exists(path): - raise TrodesAnimalInfoError('Animal {}, path ({}) exists but is not a directory.'.format(self.anim_name, - path)) - else: - raise TrodesAnimalInfoError('Animal {}, path ({}) does not exist.'.format(self.anim_name, - path)) + raise TrodesAnimalInfoError(f'Animal {self.anim_name,}, path ({path}) exists but is not a directory.') + + raise TrodesAnimalInfoError(f'Animal {self.anim_name,}, path ({path}) does not exist.') return path @@ -571,8 +554,7 @@ def _get_preprocessing_date_data_path_df(date_path_dict): date_path_entry.path])), ignore_index=True) except TrodesDataFormatError: - logger.warn(('Invalid folder name in preprocessing folder date ({}) folder ({}), ignoring.'. - format(date, date_path_entry.name))) + logger.warning(f'Invalid folder name in preprocessing folder date ({date}) folder ({date_path_entry}), ignoring.') # sort and reindex paths full_data_paths = full_data_paths.sort_values( ['date', 'epoch', 'label_ext', 'datatype']).reset_index(drop=True) @@ -580,7 +562,7 @@ def _get_preprocessing_date_data_path_df(date_path_dict): return full_data_paths @staticmethod - def _get_extracted_file_list(path, ExtractedFileParser=TrodesRawFileNameParser): + def _get_extracted_file_list(path, ExtractedFileParser=TrodesRawFileNameParser): # pylint: disable=invalid-name dir_entries = os.scandir(path) file_list = [] for dir_entry in dir_entries: @@ -589,10 +571,7 @@ def _get_extracted_file_list(path, ExtractedFileParser=TrodesRawFileNameParser): filename_parser = ExtractedFileParser(dir_entry.name) file_list.append((filename_parser, dir_entry.path)) except TrodesDataFormatError: - logger.warn('File ({}) does not match file parser ({}). Skipping.'. - format(dir_entry.path, - ExtractedFileParser.__name__), - TrodesDataFormatWarning) + logger.warning(f'File ({dir_entry.path}) does not match file parser ({ExtractedFileParser.__name__}). Skipping.', TrodesDataFormatWarning) return file_list @staticmethod @@ -622,25 +601,22 @@ def _get_day_dirs(anim_path): TrodesAnimalInfo._expand_str_date(anim_dir_entry.name) anim_day_paths[anim_dir_entry.name] = anim_dir_entry.path except ValueError: - logger.warn(('animal path ({}) contains a data directory ({}) ' - 'that does not conform to date format %Y%m%d.'). - format(anim_path, anim_dir_entry.name)) + logger.warning((f'animal path ({anim_path}) contains a data directory ({anim_dir_entry.name}) ' + 'that does not conform to date format %Y%m%d.')) except FileNotFoundError: - logger.warn(('anim path ({}) does not exist.'.format( - anim_path))) + logger.warning(f'anim path ({anim_path}) does not exist.') return anim_day_paths @staticmethod - def _get_rec_paths(path, RawFileNameParser=TrodesRawFileNameParser): + def _get_rec_paths(path, RawFileNameParser=TrodesRawFileNameParser): # pylint: disable=invalid-name, unused-argument anim_rec_paths = [] - for path in Path(path).glob('**/*.rec'): + for path in Path(path).glob('**/*.rec'): # pylint: disable=redefined-argument-from-local try: anim_rec_paths.append( (TrodesRawFileNameParser(path.name), str(path))) except TrodesDataFormatError: - logger.warn(f'Invalid trodes rec filename ({str(path.parent)}),' - ' cannot be parsed, skipping.') + logger.warning(f'Invalid trodes rec filename ({str(path.parent)}) cannot be parsed, skipping.') return anim_rec_paths @@ -651,99 +627,92 @@ def _get_trodes_version(path): for rec_file in rec_files]))[0] @staticmethod - def _get_video_tracking_paths(path, RawFileNameParser=TrodesRawFileNameParser): + def _get_video_tracking_paths(path, RawFileNameParser=TrodesRawFileNameParser): # pylint: disable=invalid-name anim_pos_paths = [] dir_entries = os.scandir(path) for dir_entry in dir_entries: if dir_entry.is_file(): - if re.match('^.*\.videoPositionTracking$', dir_entry.name): + if re.match('^.*\.videoPositionTracking$', dir_entry.name): # pylint: disable=anomalous-backslash-in-string try: trodes_filename_parsed = RawFileNameParser( dir_entry.name) anim_pos_paths.append( (trodes_filename_parsed, dir_entry.path)) except TrodesDataFormatError: - logger.warn(('Invalid trodes videoPositionTracking filename ({}), ' - 'cannot be parsed, skipping.'). - format(dir_entry.path)) + logger.warning(f'Invalid trodes videoPositionTracking filename ({dir_entry.path}) cannot be parsed, skipping.') return anim_pos_paths @staticmethod - def _get_h264_paths(path, RawFileNameParser=TrodesRawFileNameParser): + def _get_h264_paths(path, RawFileNameParser=TrodesRawFileNameParser): # pylint: disable=invalid-name, unused-argument anim_h264_paths = [] - for path in Path(path).glob('**/*.h264'): + for path in Path(path).glob('**/*.h264'): # pylint: disable=redefined-argument-from-local try: anim_h264_paths.append( (TrodesRawFileNameParser(path.name), str(path))) except TrodesDataFormatError: - logger.warn(f'Invalid trodes h264 filename ({str(path.parent)}), ' + logger.warning(f'Invalid trodes h264 filename ({str(path.parent)}), ' 'cannot be parsed, skipping.') return anim_h264_paths @staticmethod - def _get_video_timestamp_paths(path, RawFileNameParser=TrodesRawFileNameParser): + def _get_video_timestamp_paths(path, RawFileNameParser=TrodesRawFileNameParser): # pylint: disable=invalid-name anim_video_times_paths = [] dir_entries = os.scandir(path) for dir_entry in dir_entries: if dir_entry.is_file(): - if re.match('^.*\.videoTimeStamps$', dir_entry.name): + if re.match('^.*\.videoTimeStamps$', dir_entry.name): # pylint: disable=anomalous-backslash-in-string try: trodes_filename_parsed = RawFileNameParser( dir_entry.name) anim_video_times_paths.append( (trodes_filename_parsed, dir_entry.path)) except TrodesDataFormatError: - logger.warn(('Invalid trodes videoTimeStamps filename ({}), ' - 'cannot be parsed, skipping.'). - format(dir_entry.path)) + logger.warning(f'Invalid trodes videoTimeStamps filename ({dir_entry.path}) cannot be parsed, skipping.') return anim_video_times_paths @staticmethod - def _get_video_hwframecount_paths(path, RawFileNameParser=TrodesRawFileNameParser): + def _get_video_hwframecount_paths(path, RawFileNameParser=TrodesRawFileNameParser): # pylint: disable=invalid-name anim_video_hwframecount_paths = [] dir_entries = os.scandir(path) for dir_entry in dir_entries: if dir_entry.is_file(): - if re.match('^.*\.videoTimeStamps\.(?:cameraHWFrameCount$|cameraHWSync)', dir_entry.name): + if re.match('^.*\.videoTimeStamps\.(?:cameraHWFrameCount$|cameraHWSync)', dir_entry.name): # pylint: disable=anomalous-backslash-in-string try: trodes_filename_parsed = RawFileNameParser( dir_entry.name) anim_video_hwframecount_paths.append( (trodes_filename_parsed, dir_entry.path)) except TrodesDataFormatError: - logger.warn(('Invalid trodes videoTimeStamps.cameraHWFrameCount filename ({}), ' - 'cannot be parsed, skipping.'). - format(dir_entry.path)) + logger.warning(f'Invalid trodes videoTimeStamps.cameraHWFrameCount filename ({dir_entry.path}), ' + 'cannot be parsed, skipping.') return anim_video_hwframecount_paths @staticmethod - def _get_trodes_comments_paths(path, RawFileNameParser=TrodesRawFileNameParser): + def _get_trodes_comments_paths(path, RawFileNameParser=TrodesRawFileNameParser): # pylint: disable=invalid-name trodes_comment_paths = [] dir_entries = os.scandir(path) for dir_entry in dir_entries: if dir_entry.is_file(): - if re.match('^.*\.trodesComments$', dir_entry.name): + if re.match('^.*\.trodesComments$', dir_entry.name): # pylint: disable=anomalous-backslash-in-string try: trodes_filename_parsed = RawFileNameParser( dir_entry.name) trodes_comment_paths.append( (trodes_filename_parsed, dir_entry.path)) except TrodesDataFormatError: - logger.warn(('Invalid trodes .trodesComments filename ({}), ' - 'cannot be parsed, skipping.'). - format(dir_entry.path)) + logger.warning(f'Invalid trodes .trodesComments filename ({dir_entry.path}), cannot be parsed, skipping.') return trodes_comment_paths @@ -770,8 +739,8 @@ def __init__(self, anim: TrodesAnimalInfo, date, epochtuple): LFP_paths = anim.preproc_LFP_paths[(anim.preproc_LFP_paths['date'] == date) & (anim.preproc_LFP_paths['epoch'] == epochtuple)] - self.LFP_data_paths = LFP_paths[LFP_paths['timestamp_file'] == False] - self.LFP_timestamp_paths = LFP_paths[LFP_paths['timestamp_file'] == True] + self.LFP_data_paths = LFP_paths[LFP_paths['timestamp_file'] == False] # pylint: disable=singleton-comparison + self.LFP_timestamp_paths = LFP_paths[LFP_paths['timestamp_file'] == True] # pylint: disable=singleton-comparison self.lfp = pd.DataFrame() for path_tup in self.LFP_data_paths.itertuples(): @@ -782,10 +751,10 @@ def __init__(self, anim: TrodesAnimalInfo, date, epochtuple): self.lfp = pd.concat([self.lfp, pd.DataFrame(lfp_bin.data, columns=single_col)], axis=1, verify_integrity=True) else: - logger.warn(('Animal ({}), date ({}), epoch ({}) ' + logger.warning(f'Animal ({anim.anim_name}), date ({date}), epoch ({epochtuple}) ' 'has a bad preprocessing path entry, ntrode or ' 'channel has a nan entry that is not a timestamp ' - 'file, skipping.').format(anim.anim_name, date, epochtuple)) + 'file, skipping.') # get default timestamp try: @@ -793,31 +762,31 @@ def __init__(self, anim: TrodesAnimalInfo, date, epochtuple): self.LFP_timestamp_paths['time_label'] == ''] orig_timestamp_path = orig_timestamp_path_entries['path'].values[0] if len(orig_timestamp_path_entries) > 1: - logger.warn(('Animal ({}), date ({}), epoch ({}) ' + logger.warning((f'Animal ({anim.anim_name}), date ({date}), epoch ({epochtuple}) ' 'has multiple original timestamp path entries, ' - 'using ({}).').format(anim.anim_name, date, epochtuple, orig_timestamp_path)) + f'using ({orig_timestamp_path}).')) orig_timestamp_bin = TrodesTimestampBinaryLoader( orig_timestamp_path) self.orig_timestamps = orig_timestamp_bin.data except (IndexError, FileNotFoundError): self.orig_timestamps = None - raise TrodesDataFormatError(('Animal ({}), date ({}), epoch ({}) ' - 'missing default timestamps file.').format(anim.anim_name, date, epochtuple)) + raise TrodesDataFormatError((f'Animal ({anim.anim_name}), date ({date}), epoch ({epochtuple}) ' + 'missing default timestamps file.')) from FileNotFoundError try: adj_timestamp_path_entries = self.LFP_timestamp_paths[ self.LFP_timestamp_paths['time_label'] == 'adj'] adj_timestamp_path = adj_timestamp_path_entries['path'].values[0] if len(adj_timestamp_path_entries) > 1: - logger.warn(('Animal ({}), date ({}), epoch ({}) ' + logger.warning(f'Animal ({anim.anim_name}), date ({date}), epoch ({epochtuple}) ' 'has multiple adjusted timestamp path entries, ' - 'using ({}).').format(anim.anim_name, date, epochtuple, adj_timestamp_path)) + f'using ({adj_timestamp_path})') adj_timestamp_bin = TrodesTimestampBinaryLoader(adj_timestamp_path) self.adj_timestamps = adj_timestamp_bin.data except (IndexError, FileNotFoundError): self.adj_timestamps = None - logger.warn(('Animal ({}), date ({}), epoch ({}) ' - 'missing adjusted timestamps file.').format(anim.anim_name, date, epochtuple), + logger.warning((f'Animal ({anim.anim_name}), date ({date}), epoch ({epochtuple}) ' + 'missing adjusted timestamps file.'), TrodesDataFormatWarning) # always use original timestamp @@ -843,16 +812,16 @@ def __init__(self, anim: TrodesAnimalInfo, date, epochtuple, time_label, paralle path_list.append(path_tup.path) ntrode_list.append(path_tup.ntrode) else: - logger.warn(('Animal ({}), date ({}), epoch ({}) ' + logger.warning(f'Animal ({anim.anim_name}), date ({date}), epoch ({epochtuple}) ' 'has a bad preprocessing path entry, ntrode ' 'has a nan entry that is not a timestamp ' - 'file, skipping.').format(anim.anim_name, date, epochtuple)) + 'file, skipping.') - p = multiprocessing.Pool(parallel_instances) - spikes_loaded = p.map(TrodesSpikeBinaryLoader, path_list, chunksize=1) + parallel_pool = multiprocessing.Pool(parallel_instances) # pylint: disable=consider-using-with + spikes_loaded = parallel_pool.map(TrodesSpikeBinaryLoader, path_list, chunksize=1) self.spikes = dict( zip(ntrode_list, [loader.spikes for loader in spikes_loaded])) - p.close() + parallel_pool.close() class TrodesPreprocessingPosEpoch: @@ -869,18 +838,16 @@ def __init__(self, anim: TrodesAnimalInfo, date, epochtuple): for path_tup in self.pos_paths.itertuples(): if path_tup.timestamp_file: if path_tup.time_label in self.timestamps: - logger.warn(('Animal ({}), date ({}), epoch ({}) ' - 'has multiple timestamps with same label, using first one.'). - format(anim.anim_name, date, epochtuple)) + logger.warning(('Animal ({anim.anim_name}), date ({date}), epoch ({epochtuple}) ' + 'has multiple timestamps with same label, using first one.')) break timestamp_bin = TrodesTimestampBinaryLoader(path_tup.path) self.timestamps[path_tup.time_label] = timestamp_bin.data elif not pd.isnull(path_tup.pos_label): # assume path is for a position file if path_tup.pos_label in self.pos: - logger.warn(('Animal ({}), date ({}), epoch ({}) ' - 'has multiple pos data with same label, using first one.'). - format(anim.anim_name, date, epochtuple)) + logger.warning(f'Animal ({anim.anim_name}), date ({date}), epoch ({epochtuple}) ' + 'has multiple pos data with same label, using first one.') break pos_bin = TrodesPosBinaryLoader(path_tup.path) self.pos[path_tup.pos_label] = pos_bin.pos @@ -903,8 +870,10 @@ def __init__(self, anim: TrodesAnimalInfo, date, epochtuple): dio_bin = TrodesDIOBinaryLoader(path_tup.path) - dio_bin.dio.columns = pd.MultiIndex.from_tuples([(path_tup.direction, int(path_tup.channel))], - names=['direction', 'channel']) + dio_bin.dio.columns = pd.MultiIndex.from_tuples( + [(path_tup.direction, int(path_tup.channel))], + names=['direction', 'channel'] + ) dio_list.append(dio_bin.dio) @@ -935,7 +904,7 @@ def _write_spike_epoch(self, date, epoch, hdf_store, time_label, parallel_instan for ntrode, spike_ntrode in spike_epoch.spikes.items(): hdf_store.put('preprocessing/EventWaveform/' + 'e{:02d}'.format(int(epoch[0])) + '/t{:02d}'.format(int(ntrode)) + '/data', - spike_ntrode, expectedrows=len(spike_ntrode)) + spike_ntrode, expectedrows=len(spike_ntrode)) # pylint: disable=consider-using-f-string def convert_pos_day(self, date): self._convert_generic_day( @@ -1092,7 +1061,7 @@ def extract_time(self, dates, epochs, export_args=(), **kwargs): dates=dates, epochs=epochs, export_args=export_args, **kwargs) - def prepare_trodescomments(self, dates, epochs, overwrite=False, use_folder_date=False, stop_error=False): + def prepare_trodescomments(self, dates, epochs, overwrite=False, use_folder_date=False, stop_error=False): # pylint: disable=too-many-locals for dir_date, epoch in itertools.product(dates, epochs): try: out_date_dir = self.trodes_anim_info.get_preprocessing_date_dir( @@ -1101,7 +1070,7 @@ def prepare_trodescomments(self, dates, epochs, overwrite=False, use_folder_date epoch_comment_file = self.trodes_anim_info.get_raw_trodescomments_path( dir_date, epoch) except KeyError as err: - raise TrodesDataFormatError(repr(err)) + raise TrodesDataFormatError(repr(err)) from err file_parser = epoch_comment_file[0] file_path = epoch_comment_file[1] @@ -1132,11 +1101,10 @@ def prepare_trodescomments(self, dates, epochs, overwrite=False, use_folder_date if stop_error: # exception should keep raised raise - else: - # exception should be converted to a warning - logger.warn(repr(err) + ' (thrown from {}:{})' - .format(sys.exc_info()[2].tb_frame.f_code.co_filename, - sys.exc_info()[2].tb_lineno)) + + # exception should be converted to a warning + logger.warning(f'{repr(err)} (thrown from {sys.exc_info()[2].tb_frame.f_code.co_filename}:{sys.exc_info()[2].tb_lineno})') + def prepare_mountain_dir(self, dates, epochs, use_folder_date=False, stop_error=False): @@ -1149,7 +1117,7 @@ def prepare_mountain_dir(self, dates, epochs, use_folder_date=False, stop_error= dir_date, epoch) except KeyError: raise TrodesDataFormatError('Rec: Date {} and epoch {} does not exist for animal {}.'. - format(dir_date, epoch, self.trodes_anim_info.anim_name)) + format(dir_date, epoch, self.trodes_anim_info.anim_name)) from None file_parser = epoch_raw_file[0] @@ -1172,16 +1140,15 @@ def prepare_mountain_dir(self, dates, epochs, use_folder_date=False, stop_error= except TrodesDataFormatError as err: if stop_error: - # exception should keep raised + # exception should be raised raise - else: - # exception should be converted to a warning - logger.warn(repr(err) + ' (thrown from {}:{})' - .format(sys.exc_info()[2].tb_frame.f_code.co_filename, - sys.exc_info()[2].tb_lineno)) - def prepare_pos_dir(self, dates, epochs, overwrite=False, use_folder_date=False, stop_error=False): - for dir_date, epoch in itertools.product(dates, epochs): + # exception should be converted to a warning + logger.warning(f'{repr(err)} thrown from {sys.exc_info()[2].tb_frame.f_code.co_filename}:{sys.exc_info()[2].tb_lineno})') + + + def prepare_pos_dir(self, dates, epochs, overwrite=False, use_folder_date=False, stop_error=False): # pylint: disable=too-many-locals, too-many-statements, too-many-branches + for dir_date, epoch in itertools.product(dates, epochs): # pylint: disable=too-many-nested-blocks try: out_date_dir = self.trodes_anim_info.get_preprocessing_date_dir( dir_date, stop_error=False) @@ -1189,10 +1156,9 @@ def prepare_pos_dir(self, dates, epochs, overwrite=False, use_folder_date=False, h264_epoch_pathset = self.trodes_anim_info.get_raw_h264_paths( dir_date, epoch) except KeyError: - raise TrodesDataFormatError('h264: Date {} and epoch {} does not exist for animal {}.'. - format(dir_date, epoch, self.trodes_anim_info.anim_name)) + raise TrodesDataFormatError(f'h264: Date {dir_date} and epoch {epoch} does not exist for animal {self.trodes_anim_info.anim_name}') from KeyError - for label_ext, (h264_name_parser, h264_path) in h264_epoch_pathset.items(): + for label_ext, (h264_name_parser) in h264_epoch_pathset.items(): # create position dir if use_folder_date: out_base_date = dir_date @@ -1212,7 +1178,7 @@ def prepare_pos_dir(self, dates, epochs, overwrite=False, use_folder_date=False, # trying to move online position files over try: - (raw_pos_filename_parser, raw_pos_path) = \ + (_, raw_pos_path) = \ self.trodes_anim_info.get_raw_pos_path(date=dir_date, epoch=epoch, label_ext=label_ext) @@ -1229,12 +1195,10 @@ def prepare_pos_dir(self, dates, epochs, overwrite=False, use_folder_date=False, except KeyError as err: # this file does not exist - logger.warn(repr(err) + ' (thrown from {}:{})' - .format(sys.exc_info()[2].tb_frame.f_code.co_filename, - sys.exc_info()[2].tb_lineno)) + logger.warning(f'{repr(err)} thrown from {sys.exc_info()[2].tb_frame.f_code.co_filename}:{sys.exc_info()[2].tb_lineno})') try: - (raw_postime_filename_parser, raw_postime_path) = \ + (_, raw_postime_path) = \ self.trodes_anim_info.get_raw_postime_path(date=dir_date, epoch=epoch, label_ext=label_ext) @@ -1251,12 +1215,10 @@ def prepare_pos_dir(self, dates, epochs, overwrite=False, use_folder_date=False, except KeyError as err: # this file does not exist - logger.warn(repr(err) + ' (thrown from {}:{})' - .format(sys.exc_info()[2].tb_frame.f_code.co_filename, - sys.exc_info()[2].tb_lineno)) + logger.warning(f'{repr(err)} (thrown from {sys.exc_info()[2].tb_frame.f_code.co_filename}:{sys.exc_info()[2].tb_lineno})') try: - (raw_poshwframecount_filename_parser, raw_poshwframecount_path) = \ + (_, raw_poshwframecount_path) = \ self.trodes_anim_info.get_raw_poshwframecount_path(date=dir_date, epoch=epoch, label_ext=label_ext + '.videoTimeStamps') @@ -1274,23 +1236,20 @@ def prepare_pos_dir(self, dates, epochs, overwrite=False, use_folder_date=False, except KeyError as err: # this file does not exist - logger.warn(repr(err) + ' (thrown from {}:{})' - .format(sys.exc_info()[2].tb_frame.f_code.co_filename, - sys.exc_info()[2].tb_lineno)) + logger.warning(f'repr(err) (thrown from {sys.exc_info()[2].tb_frame.f_code.co_filename}:{sys.exc_info()[2].tb_lineno})') except TrodesDataFormatError as err: if stop_error: # exception should keep raised raise - else: - # exception should be converted to a warning - logger.warn(repr(err) + ' (thrown from {}:{})' - .format(sys.exc_info()[2].tb_frame.f_code.co_filename, - sys.exc_info()[2].tb_lineno)) + + # exception should be converted to a warning + logger.warning(f'repr(err) (thrown from {sys.exc_info()[2].tb_frame.f_code.co_filename}:{sys.exc_info()[2].tb_lineno})') def _extract_rec_generic(self, export_cmd, export_dir_ext, dates, epochs, export_args=(), overwrite=False, stop_error=False, use_folder_date=False, parallel_instances=1, use_day_config=True): + # pylint: disable=too-many-branches, too-many-statements, too-many-locals """ Args: export_cmd (str): @@ -1327,8 +1286,7 @@ def _extract_rec_generic(self, export_cmd, export_dir_ext, epoch_raw_file = self.trodes_anim_info.get_raw_rec_path( dir_date, epoch) except KeyError: - raise TrodesDataFormatError('Rec: Date {} and epoch {} does not exist for animal {}.'. - format(dir_date, epoch, self.trodes_anim_info.anim_name)) + raise TrodesDataFormatError(f'Rec: Date {dir_date} and epoch {epoch} does not exist for animal {self.trodes_anim_info.anim_name}.') from KeyError # sort files epoch_raw_file = sorted(epoch_raw_file, key=lambda x: x[1]) @@ -1343,15 +1301,13 @@ def _extract_rec_generic(self, export_cmd, export_dir_ext, if file_parser.date != dir_date: if use_folder_date: - logger.warn(('For rec file ({}) the date field does not match ' - 'the folder date ({}). Forcing file to use folder date'). - format(file_parser.filename, dir_date), TrodesDataFormatWarning) + logger.warning(f'For rec file ({file_parser.filename}) the date field does not match the folder date ({dir_date}). Forcing file to use folder date', + TrodesDataFormatWarning) else: - logger.warn(('For rec file ({}) the date field does not match ' - 'the folder date ({}). This should be fixed or could have ' + logger.warning((f'For rec file ({file_parser.filename}) the date field does not match ' + f'the folder date ({dir_date}). This should be fixed or could have ' 'unintended parsing consequences. Will proceed by maintaining ' - 'folder date and using rec file date in extracted files.'). - format(file_parser.filename, dir_date), TrodesDataFormatWarning) + 'folder date and using rec file date in extracted files.'), TrodesDataFormatWarning) if use_folder_date: out_base_date = dir_date @@ -1410,7 +1366,7 @@ def _extract_rec_generic(self, export_cmd, export_dir_ext, out_epoch_dir, out_base_filename + '.' + cmd_type + '.log') - out_cmd_log_file = open(out_cmd_log_filename, 'w') + out_cmd_log_file = open(out_cmd_log_filename, 'w') # pylint: disable=consider-using-with, unspecified-encoding # prepend the call command and argument to the log file out_cmd_log_file.write(' '.join(export_call)) out_cmd_log_file.write('\n') @@ -1423,31 +1379,30 @@ def _extract_rec_generic(self, export_cmd, export_dir_ext, next_cmd_id, export_call)) subprocess_pool[next_cmd_id] = ( - subprocess.Popen(export_call, - stdout=out_cmd_log_file, - stderr=out_cmd_log_file), + subprocess.Popen( # pylint: disable=consider-using-with + export_call, + stdout=out_cmd_log_file, + stderr=out_cmd_log_file + ), out_cmd_log_filename) next_cmd_id += 1 - except TrodesDataFormatError as err: + except TrodesDataFormatError as err: # pylint: disable=unused-variable if stop_error: # exception should keep raised raise - else: - # exception should be converted to a warning - logger.warn(repr(err) + ' (thrown from {}:{})' - .format(sys.exc_info()[2].tb_frame.f_code.co_filename, - sys.exc_info()[2].tb_lineno)) + + # exception should be converted to a warning + logger.warning(f'repr(err) (thrown from {sys.exc_info()[2].tb_frame.f_code.co_filename}:{sys.exc_info()[2].tb_lineno})') # wait for all commands to finish just_terminated = self._wait_subprocess_pool( subprocess_pool, wait_pool_size=0) terminated_processes.update(just_terminated) - for cmd_key, (extract_proc, cmd_log_file) in terminated_processes.items(): + for _, (extract_proc, __) in terminated_processes.items(): if extract_proc.poll() != 0: - logger.warn('Running export command ({}) failed with return code {}'. - format(extract_proc.args, extract_proc.poll()), TrodesDataFormatWarning) + logger.warning(f'Running export command ({extract_proc.args}) failed with return code {extract_proc.poll()}', TrodesDataFormatWarning) @staticmethod def _assemble_export_base_name(date, anim_name, epochlist, label, label_ext): @@ -1473,8 +1428,8 @@ def _wait_subprocess_pool(subprocess_pool, wait_pool_size): print('(ID: {}) Done running {}'.format( cmd_key, extract_proc.args)) # print log file - with open(cmd_log_filename, 'r') as f: - for line in f: + with open(cmd_log_filename, 'r') as file_name: # pylint: disable=unspecified-encoding + for line in file_name: print('(ID: {}) '.format(cmd_key) + line, end='') return terminated_processes @@ -1497,13 +1452,12 @@ def get_trodes_version(rec_file_name): def get_trodes_version_from_path(): try: - result = str(subprocess.run(['exportmda', '-v'], capture_output=True) - .stdout) + result = str(subprocess.run(['exportmda', '-v'], capture_output=True).stdout) # pylint: disable=subprocess-run-check except FileNotFoundError: - result = str(subprocess.run(['trodesexport', '-v'], capture_output=True) - .stdout) - version = (result + result = str(subprocess.run(['trodesexport', '-v'], capture_output=True).stdout) # pylint: disable=subprocess-run-check + version = (result # pylint: disable=use-maxsplit-arg .split('\\n')[0] .split(' ')[2] .split('.')) + return int(version[0]), int(version[1]), int(version[2]) diff --git a/unit_test/__init__.py b/unit_test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/unit_test/conftest.py b/unit_test/conftest.py new file mode 100644 index 0000000..e69de29 diff --git a/unit_test/test_adjust_timestamps.py b/unit_test/test_adjust_timestamps.py new file mode 100644 index 0000000..902dca0 --- /dev/null +++ b/unit_test/test_adjust_timestamps.py @@ -0,0 +1,68 @@ +from unittest.mock import MagicMock, patch +import pytest +from faker import Faker +from rec_to_binaries.adjust_timestamps import ( + _label_time_chunks, + fix_timestamp_lag +) + + +@patch('numpy.asarray') +@patch('numpy.diff') +@patch('numpy.insert') +@patch('numpy.cumsum') +def test___label_time_chunks(mock_cumsum, mock_insert, mock_diff, mock_asarray): + test_argument = 5 + test_diff_val = 10 + test_insert_val = 20 + mock_diff.return_value = test_diff_val + mock_insert.return_value = test_insert_val + + _label_time_chunks(test_argument) + + assert mock_asarray.called + assert mock_diff.called + assert mock_insert.called + assert mock_asarray.call_args[0][0] == test_argument + assert mock_cumsum.called + assert mock_cumsum.call_args[0][0] == test_insert_val + + +@pytest.mark.parametrize('names', ['', 'systime']) +@patch('rec_to_binaries.adjust_timestamps._insert_new_data') +@patch('rec_to_binaries.adjust_timestamps.readTrodesExtractedDataFile') +@patch('rec_to_binaries.adjust_timestamps.logger') +@patch('rec_to_binaries.adjust_timestamps.write_trodes_extracted_datafile') +@patch('pandas.DataFrame') +@patch('rec_to_binaries.adjust_timestamps.infer_systime') +def test_fix_timestamp_lag( + mock_infer_systime, + mock_pandas_DataFrame, + mock_write_trodes_extracted_datafile, + mock_logger, + mock_readTrodesExtractedDataFile, + mock__insert_new_data, + names + ): + #fix_timestamp_lag + fake = Faker() + mock_logger.return_value.warn.return_value = fake.numerify() + mock__insert_new_data.return_value = fake.numerify() + mock_write_trodes_extracted_datafile.return_value = fake.word() + fake_continuoustime_filename = fake.file_name() + magicMock = MagicMock() + fake_data_file = MagicMock() + fake_data_file_return = MagicMock() + fake_data_file_return.names = names + fake_data_file.return_value = { 'data': fake_data_file } + + mock_readTrodesExtractedDataFile.return_value = fake_data_file + mock_infer_systime.return_value = fake.word() + + fix_timestamp_lag(fake_continuoustime_filename) + + assert mock_readTrodesExtractedDataFile.called + assert 'systime' in names or not mock_logger.called + assert not 'systime' in names or not mock_pandas_DataFrame.called + assert mock__insert_new_data.called + assert mock_write_trodes_extracted_datafile.called diff --git a/unit_test/test_binary_utils.py b/unit_test/test_binary_utils.py new file mode 100644 index 0000000..1a8f1e9 --- /dev/null +++ b/unit_test/test_binary_utils.py @@ -0,0 +1,70 @@ +from rec_to_binaries.binary_utils import ( + TrodesBinaryReader, +) +from unittest.mock import MagicMock, patch, mock_open +import pytest +from faker import Faker + + +# @pytest.mark.parametrize('fake_read_line', [b'\n']) +# @patch('rec_to_binaries.binary_utils.TrodesBinaryReader') +# def test_TrodesBinaryReader__init__(mock_TrodesBinaryReader, fake_read_line): +# faker = Faker() +# fake_file_path = faker.file_path() +# m = mock_open() +# breakpoint(); +# mock_TrodesBinaryReader.file.readline.return_value = faker.numerify() +# mock_TrodesBinaryReader.file = [] +# with patch('__main__.open', m, create=True): +# with open(faker.word(), 'w') as h: +# h.readline(faker.numerify()) + +# trodesBinaryReader = TrodesBinaryReader(fake_file_path) + + +# assert False + + + +# >>> with patch('__main__.open', m, create=True): +# ... with open('foo', 'w') as h: +# ... h.write('some stuff') + +# class TrodesBinaryReader: +# def __init__(self, path): +# self.path = path +# with open(path, 'rb') as file: +# # reading header +# # read first line to make sure its a trodes binary header +# line = file.readline() +# if line != b'\n': +# raise TrodesBinaryFormatError( +# 'File ({}) does not start with Trodes header ' +# .format(path)) +# # read header not including start and end tags +# self.header_params = {} +# for line_no, line in enumerate(file): +# if line == b'\n': +# break +# if line_no > 1000: +# raise TrodesBinaryFormatError('File ({}) header over 1000 lines without ' +# .format(path)) +# line_split = line.decode('utf-8').split(':', 1) +# self.header_params[line_split[0]] = line_split[1].strip() + +# self.data_start_byte = file.tell() + +# >>> m = mock_open() +# >>> with patch('__main__.open', m, create=True): +# ... with open('foo', 'w') as h: +# ... h.write('some stuff') + +# >>> m.mock_calls +# [call('foo', 'w'), +# call().__enter__(), +# call().write('some stuff'), +# call().__exit__(None, None, None)] +# >>> m.assert_called_once_with('foo', 'w') +# >>> handle = m() +# >>> handle.write.assert_called_once_with('some stuff') +