From dc13b1fa1ef61d86f0e73a8b1a2e45bdbf2142ee Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Tue, 13 Jun 2017 09:24:31 +0200 Subject: [PATCH 01/13] refact(buffer): needless np.int64 instead of regular python ints + Python's ints have arbitrary digits, are not just 32bit. --- vitables/vttables/buffer.py | 36 ++++++++++++---------------------- vitables/vttables/leaf_view.py | 11 ++++------- 2 files changed, 16 insertions(+), 31 deletions(-) diff --git a/vitables/vttables/buffer.py b/vitables/vttables/buffer.py index dcbde21a..f542d7ff 100644 --- a/vitables/vttables/buffer.py +++ b/vitables/vttables/buffer.py @@ -92,18 +92,14 @@ def __init__(self, leaf): # The maximum number of rows to be read from the data source self.chunk_size = 10000 - # The length of the dimension that is going to be read. It - # is an int64. + # The length of the dimension that is going to be read. self.leaf_numrows = self.leafNumberOfRows() if self.leaf_numrows <= self.chunk_size: self.chunk_size = self.leaf_numrows # The numpy array where read data will be stored self.chunk = numpy.array([]) - - # The document row where the current chunk of data starts. - # It must be an int64 in order to address spaces bigger than 2**32 - self.start = numpy.array(0, dtype=numpy.int64) + self.start = 0 # The method used for reading data depends on the kind of node. # Setting the reader method at initialization time increases the @@ -160,7 +156,7 @@ def leafNumberOfRows(self): else: nrows = self.data_source.nrows - return numpy.array(nrows, dtype=numpy.int64) + return nrows def getReadParameters(self, start, buffer_size): """ @@ -169,14 +165,13 @@ def getReadParameters(self, start, buffer_size): :Parameters: - `start`: the document row that is the first row of the chunk. - It *must* be a 64 bits integer. - `buffer_size`: the buffer size, i.e. the number of rows to be read. :Returns: a tuple with tested values for the parameters of the read method """ - first_row = numpy.array(0, dtype=numpy.int64) + first_row = 0 last_row = self.leaf_numrows # When scrolling up we must keep start value >= first_row @@ -203,7 +198,7 @@ def isDataSourceReadable(self): """ readable = True - start, stop = self.getReadParameters(numpy.array(0, dtype=numpy.int64), + start, stop = self.getReadParameters(0, self.chunk_size) try: self.data_source.read(start, stop) @@ -240,7 +235,6 @@ def readBuffer(self, start, buffer_size): :Parameters: - `start`: the document row that is the first row of the chunk. - It *must* be a 64 bits integer. - `buffer_size`: the buffer size, i.e. the number of rows to be read. """ @@ -273,7 +267,7 @@ def scalarCell(self, row, col): :Parameters: - - `row`: the row to which the cell belongs. It is a 64 bits integer + - `row`: the row to which the cell belongs. - `col`: the column to wich the cell belongs :Returns: the cell at position `(row, col)` of the document @@ -300,16 +294,14 @@ def vectorCell(self, row, col): :Parameters: - - `row`: the row to which the cell belongs. It is a 64 bits integer + - `row`: the row to which the cell belongs. - `col`: the column to wich the cell belongs :Returns: the cell at position `(row, col)` of the document """ # We must shift the row value by self.start units in order to - # get the right chunk element. Note that indices of chunk - # needn't to be int64 because they are indexing a fixed size, - # small chunk of data (see ctor docstring). + # get the right chunk element. # chunk = [row0, row1, row2, ..., rowN] # and columns can be read from a given row using indexing notation try: @@ -327,16 +319,14 @@ def EArrayCell(self, row, col): :Parameters: - - `row`: the row to which the cell belongs. It is a 64 bits integer + - `row`: the row to which the cell belongs. - `col`: the column to wich the cell belongs :Returns: the cell at position `(row, col)` of the document """ # We must shift the row value by self.start units in order to - # get the right chunk element. Note that indices of chunk - # needn't to be int64 because they are indexing a fixed size, - # small chunk of data (see ctor docstring). + # get the right chunk element. # chunk = [row0, row1, row2, ..., rowN] # and columns can be read from a given row using indexing notation # TODO: this method should be improved as it requires to read the @@ -356,16 +346,14 @@ def arrayCell(self, row, col): :Parameters: - - `row`: the row to which the cell belongs. It is a 64 bits integer + - `row`: the row to which the cell belongs. - `col`: the column to wich the cell belongs :Returns: the cell at position `(row, col)` of the document """ # We must shift the row value by self.start units in order to get the - # right chunk element. Note that indices of chunk needn't to be - # int64 because they are indexing a fixed size, small chunk of - # data (see ctor docstring). + # right chunk element. # For arrays we have # chunk = [row0, row1, row2, ..., rowN] # and columns can be read from a given row using indexing notation diff --git a/vitables/vttables/leaf_view.py b/vitables/vttables/leaf_view.py index 1ea107d3..419e849d 100644 --- a/vitables/vttables/leaf_view.py +++ b/vitables/vttables/leaf_view.py @@ -144,8 +144,7 @@ def mapSlider2Leaf(self): # rows and row equals to value interval_size = 1 if self.max_value < self.leaf_numrows: - interval_size = numpy.rint(numpy.array( - self.leaf_numrows/self.max_value, dtype=numpy.int64)) + interval_size = round(self.leaf_numrows / self.max_value) return interval_size @@ -166,8 +165,7 @@ def syncView(self): elif fv_label == 1 : self.tricky_vscrollbar.setValue(0) else : - value = numpy.rint(numpy.array(fv_label/self.interval_size, - dtype=numpy.float64)) + value = round(fv_label / self.interval_size) self.tricky_vscrollbar.setValue(value) @@ -372,7 +370,7 @@ def dragSlider(self): value = self.max_value row = self.leaf_numrows - 1 else : - row = numpy.array(self.interval_size*value, dtype=numpy.int64) + row = self.interval_size * value # top buffer fault condition if row < model.rbuffer.start: @@ -465,8 +463,7 @@ def wheelEvent(self, event): # has been rotated by 15 degrees. It *seems* that every eight of # degree corresponds to a distance of 1 pixel. delta = event.angleDelta().y() - self.wheel_step = \ - numpy.rint(abs(delta)/height).astype(numpy.int64) - 1 + self.wheel_step = round(abs(delta) / height) - 1 if delta < 0: self.wheelDown(event) else: From 4779c3b380a25006a09b113206e2cddb7b803366 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Tue, 13 Jun 2017 17:54:10 +0200 Subject: [PATCH 02/13] refact(buffer): don't re-offset with `buffer.start` when reading cells + The `buffer.start` offset was added by models/timeseries to describe coordinates against the full table; buffer internally was substracting it back. + Change reduces a bit coupling to internals of buffer (Demeter law). --- vitables/plugins/timeseries/time_series.py | 6 ++---- vitables/vttables/buffer.py | 14 +++++++------- vitables/vttables/datasheet.py | 2 +- vitables/vttables/leaf_model.py | 3 +-- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/vitables/plugins/timeseries/time_series.py b/vitables/plugins/timeseries/time_series.py index 83f9a8aa..cd31dac5 100644 --- a/vitables/plugins/timeseries/time_series.py +++ b/vitables/plugins/timeseries/time_series.py @@ -329,8 +329,7 @@ def table_data(self, index, role=QtCore.Qt.DisplayRole): if not index.isValid() or \ not (0 <= index.row() < self.numrows): return None - cell = self.rbuffer.getCell(self.rbuffer.start + index.row(), - index.column()) + cell = self.rbuffer.getCell(index.row(), index.column()) if role == QtCore.Qt.DisplayRole: if index.column() in self.ts_cols: return self.tsFormatter(cell) @@ -356,8 +355,7 @@ def array_data(self, index, role=QtCore.Qt.DisplayRole): if not index.isValid() or \ not (0 <= index.row() < self.numrows): return None - cell = self.rbuffer.getCell(self.rbuffer.start + index.row(), - index.column()) + cell = self.rbuffer.getCell(index.row(), index.column()) if role == QtCore.Qt.DisplayRole: return self.tsFormatter(cell) elif role == QtCore.Qt.TextAlignmentRole: diff --git a/vitables/vttables/buffer.py b/vitables/vttables/buffer.py index f542d7ff..083ede38 100644 --- a/vitables/vttables/buffer.py +++ b/vitables/vttables/buffer.py @@ -300,12 +300,12 @@ def vectorCell(self, row, col): :Returns: the cell at position `(row, col)` of the document """ - # We must shift the row value by self.start units in order to + # The row-coordinate must be shifted by model.start units in order to # get the right chunk element. # chunk = [row0, row1, row2, ..., rowN] # and columns can be read from a given row using indexing notation try: - return self.chunk[int(row - self.start)] + return self.chunk[row] except IndexError: self.logger.error('IndexError! buffer start: {0} row, column: ' '{1}, {2}'.format(self.start, row, col)) @@ -325,14 +325,14 @@ def EArrayCell(self, row, col): :Returns: the cell at position `(row, col)` of the document """ - # We must shift the row value by self.start units in order to + # The row-coordinate must be shifted by model.start units in order to # get the right chunk element. # chunk = [row0, row1, row2, ..., rowN] # and columns can be read from a given row using indexing notation # TODO: this method should be improved as it requires to read the # whola array keeping the read data in memory try: - return self.data_source.read()[int(row - self.start)] + return self.data_source.read()[row] except IndexError: self.logger.error('IndexError! buffer start: {0} row, column: ' '{1}, {2}'.format(self.start, row, col)) @@ -352,8 +352,8 @@ def arrayCell(self, row, col): :Returns: the cell at position `(row, col)` of the document """ - # We must shift the row value by self.start units in order to get the - # right chunk element. + # The row-coordinate must be shifted by model.start units in order to + # get the right chunk element. # For arrays we have # chunk = [row0, row1, row2, ..., rowN] # and columns can be read from a given row using indexing notation @@ -361,7 +361,7 @@ def arrayCell(self, row, col): # chunk = [nestedrecord0, nestedrecord1, ..., nestedrecordN] # and fields can be read from nestedrecordJ using indexing notation try: - return self.chunk[int(row - self.start)][col] + return self.chunk[row][col] except IndexError: self.logger.error('IndexError! buffer start: {0} row, column: ' '{1}, {2}'.format(self.start, row, col)) diff --git a/vitables/vttables/datasheet.py b/vitables/vttables/datasheet.py index c462e66e..833bbb98 100644 --- a/vitables/vttables/datasheet.py +++ b/vitables/vttables/datasheet.py @@ -154,7 +154,7 @@ def zoomCell(self, index): row = index.row() column = index.column() tmodel = index.model() - data = tmodel.rbuffer.getCell(tmodel.rbuffer.start + row, column) + data = tmodel.rbuffer.getCell(row, column) # The title of the zoomed view node = self.dbt_leaf diff --git a/vitables/vttables/leaf_model.py b/vitables/vttables/leaf_model.py index 488673ce..c07c5f60 100644 --- a/vitables/vttables/leaf_model.py +++ b/vitables/vttables/leaf_model.py @@ -162,8 +162,7 @@ def data(self, index, role=QtCore.Qt.DisplayRole): if not index.isValid() or not (0 <= index.row() < self.numrows): return None - cell = self.rbuffer.getCell(self.rbuffer.start + index.row(), - index.column()) + cell = self.rbuffer.getCell(index.row(), index.column()) if role == QtCore.Qt.DisplayRole: return self.formatContent(cell) elif role == QtCore.Qt.TextAlignmentRole: From 8a4e0d0d7bb720f3f62f6c337ac3e2b357a973e2 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Tue, 13 Jun 2017 17:58:36 +0200 Subject: [PATCH 03/13] fix(buffer): speedup table readability-check + Reading a full chunk (1000 rows) from a compressed HDF5 file would be a waste of CPU. + Also decouple readability-check from chunk-size/start value --- vitables/vttables/buffer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vitables/vttables/buffer.py b/vitables/vttables/buffer.py index 083ede38..dda2feea 100644 --- a/vitables/vttables/buffer.py +++ b/vitables/vttables/buffer.py @@ -198,12 +198,11 @@ def isDataSourceReadable(self): """ readable = True - start, stop = self.getReadParameters(0, - self.chunk_size) try: - self.data_source.read(start, stop) + self.data_source.read(0, 1) except tables.HDF5ExtError: readable = False + ## TODO: Fix error msg to include exception. self.logger.error( translate('Buffer', """Problems reading records. The dataset """ @@ -247,6 +246,7 @@ def readBuffer(self, start, buffer_size): # array returned by EArray.read() will have only 2 rows data = self.data_source.read(start, stop) except tables.HDF5ExtError: + ## TODO: Fix error msg to include exception. self.logger.error( translate('Buffer', """\nError: problems reading records. """ """The dataset maybe corrupted.""", From 1991ca2df459fd0f59bbc8e57531aac605886e88 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Tue, 13 Jun 2017 17:58:36 +0200 Subject: [PATCH 04/13] refact(table): do not load cell-values if not `display` QtRole + Speedup model by avoiding useless data-loading. --- vitables/plugins/timeseries/time_series.py | 30 ++++++++++++---------- vitables/vttables/leaf_model.py | 23 +++++++++-------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/vitables/plugins/timeseries/time_series.py b/vitables/plugins/timeseries/time_series.py index cd31dac5..f07219db 100644 --- a/vitables/plugins/timeseries/time_series.py +++ b/vitables/plugins/timeseries/time_series.py @@ -326,18 +326,19 @@ def table_data(self, index, role=QtCore.Qt.DisplayRole): - `role`: the role being returned """ - if not index.isValid() or \ - not (0 <= index.row() < self.numrows): + if not index.isValid() or not (0 <= index.row() < self.numrows): return None - cell = self.rbuffer.getCell(index.row(), index.column()) + if role == QtCore.Qt.DisplayRole: + cell = self.rbuffer.getCell(index.row(), index.column()) if index.column() in self.ts_cols: return self.tsFormatter(cell) return self.formatContent(cell) - elif role == QtCore.Qt.TextAlignmentRole: - return int(QtCore.Qt.AlignLeft|QtCore.Qt.AlignTop) - else: - return None + + if role == QtCore.Qt.TextAlignmentRole: + return QtCore.Qt.AlignLeft | QtCore.Qt.AlignTop + + return None def array_data(self, index, role=QtCore.Qt.DisplayRole): @@ -352,16 +353,17 @@ def array_data(self, index, role=QtCore.Qt.DisplayRole): - `role`: the role being returned """ - if not index.isValid() or \ - not (0 <= index.row() < self.numrows): + if not index.isValid() or not (0 <= index.row() < self.numrows): return None - cell = self.rbuffer.getCell(index.row(), index.column()) + if role == QtCore.Qt.DisplayRole: + cell = self.rbuffer.getCell(index.row(), index.column()) return self.tsFormatter(cell) - elif role == QtCore.Qt.TextAlignmentRole: - return int(QtCore.Qt.AlignLeft|QtCore.Qt.AlignTop) - else: - return None + + if role == QtCore.Qt.TextAlignmentRole: + return QtCore.Qt.AlignLeft|QtCore.Qt.AlignTop + + return None def timeFormatter(self): diff --git a/vitables/vttables/leaf_model.py b/vitables/vttables/leaf_model.py index c07c5f60..cb6b6f0b 100644 --- a/vitables/vttables/leaf_model.py +++ b/vitables/vttables/leaf_model.py @@ -133,20 +133,22 @@ def headerData(self, section, orientation, role): # The section alignment if role == QtCore.Qt.TextAlignmentRole: if orientation == QtCore.Qt.Horizontal: - return int(QtCore.Qt.AlignLeft|QtCore.Qt.AlignVCenter) - return int(QtCore.Qt.AlignRight|QtCore.Qt.AlignVCenter) + return QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter + return QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter + if role != QtCore.Qt.DisplayRole: return None - # The section label for horizontal header + + ## Columns-labels if orientation == QtCore.Qt.Horizontal: # For tables horizontal labels are column names, for arrays # the section numbers are used as horizontal labels if hasattr(self.data_source, 'description'): return str(self.data_source.colnames[section]) return str(section) - # The section label for vertical header. This is a 64 bits integer - return str(self.rbuffer.start + section) + ## Rows-labels + return str(self.rbuffer.start + section) def data(self, index, role=QtCore.Qt.DisplayRole): """Returns the data stored under the given role for the item @@ -162,14 +164,15 @@ def data(self, index, role=QtCore.Qt.DisplayRole): if not index.isValid() or not (0 <= index.row() < self.numrows): return None - cell = self.rbuffer.getCell(index.row(), index.column()) + if role == QtCore.Qt.DisplayRole: + cell = self.rbuffer.getCell(index.row(), index.column()) return self.formatContent(cell) - elif role == QtCore.Qt.TextAlignmentRole: - return int(QtCore.Qt.AlignLeft|QtCore.Qt.AlignTop) - else: - return None + if role == QtCore.Qt.TextAlignmentRole: + return QtCore.Qt.AlignLeft | QtCore.Qt.AlignTop + + return None def columnCount(self, index=QtCore.QModelIndex()): """The number of columns of the given model index. From 5f6b563c960b70915c7b6be67a498c0dd4667fb6 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Tue, 13 Jun 2017 22:12:17 +0200 Subject: [PATCH 05/13] refact(table): rename data_source-->leaf --- vitables/vttables/buffer.py | 18 +++++++++--------- vitables/vttables/leaf_model.py | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/vitables/vttables/buffer.py b/vitables/vttables/buffer.py index dda2feea..90013b42 100644 --- a/vitables/vttables/buffer.py +++ b/vitables/vttables/buffer.py @@ -87,7 +87,7 @@ def __init__(self, leaf): self.logger = logging.getLogger(__name__) self.logger.setLevel(logging.INFO) - self.data_source = leaf + self.leaf = leaf # The maximum number of rows to be read from the data source self.chunk_size = 10000 @@ -142,19 +142,19 @@ def leafNumberOfRows(self): :Returns: the size of the first dimension of the document """ - shape = self.data_source.shape + shape = self.leaf.shape if shape is None: # Node is not a Leaf or there was problems getting the shape nrows = 0 elif shape == (): # Node is a rank 0 array (e.g. numpy.array(5)) nrows = 1 - elif isinstance(self.data_source, tables.EArray): + elif isinstance(self.leaf, tables.EArray): # Warning: the number of rows of an EArray, ea, can be different # from the number of rows of the numpy array ea.read() - nrows = self.data_source.shape[0] + nrows = self.leaf.shape[0] else: - nrows = self.data_source.nrows + nrows = self.leaf.nrows return nrows @@ -199,7 +199,7 @@ def isDataSourceReadable(self): readable = True try: - self.data_source.read(0, 1) + self.leaf.read(0, 1) except tables.HDF5ExtError: readable = False ## TODO: Fix error msg to include exception. @@ -210,7 +210,7 @@ def isDataSourceReadable(self): """Check that it is installed in your system, """ """please.""", 'A dataset readability error').format( - self.data_source.filters.complib)) + self.leaf.filters.complib)) except ValueError as e: readable = False self.logger.error( @@ -244,7 +244,7 @@ def readBuffer(self, start, buffer_size): # Warning: in a EArray with shape (2,3,3) and extdim attribute # being 1, the read method will have 3 rows. However, the numpy # array returned by EArray.read() will have only 2 rows - data = self.data_source.read(start, stop) + data = self.leaf.read(start, stop) except tables.HDF5ExtError: ## TODO: Fix error msg to include exception. self.logger.error( @@ -332,7 +332,7 @@ def EArrayCell(self, row, col): # TODO: this method should be improved as it requires to read the # whola array keeping the read data in memory try: - return self.data_source.read()[row] + return self.leaf.read()[row] except IndexError: self.logger.error('IndexError! buffer start: {0} row, column: ' '{1}, {2}'.format(self.start, row, col)) diff --git a/vitables/vttables/leaf_model.py b/vitables/vttables/leaf_model.py index cb6b6f0b..8c599be1 100644 --- a/vitables/vttables/leaf_model.py +++ b/vitables/vttables/leaf_model.py @@ -54,7 +54,7 @@ def __init__(self, rbuffer, parent=None): """ # The model data source (a PyTables/HDF5 leaf) and its access buffer - self.data_source = rbuffer.data_source + self.leaf = rbuffer.leaf self.rbuffer = rbuffer # The number of digits of the last row @@ -74,14 +74,14 @@ def __init__(self, rbuffer, parent=None): # The dataset number of columns doesn't use to be large so, we don't # need set a maximum as we did with rows. The whole set of columns # are displayed - if isinstance(self.data_source, tables.Table): + if isinstance(self.leaf, tables.Table): # Leaf is a PyTables table - self.numcols = len(self.data_source.colnames) - elif isinstance(self.data_source, tables.EArray): + self.numcols = len(self.leaf.colnames) + elif isinstance(self.leaf, tables.EArray): self.numcols = 1 else: # Leaf is some kind of PyTables array - shape = self.data_source.shape + shape = self.leaf.shape if len(shape) > 1: # The leaf will be displayed as a bidimensional matrix self.numcols = shape[1] @@ -98,9 +98,9 @@ def __init__(self, rbuffer, parent=None): # Time series (if they are found) are formatted transparently # via the time_series.py plugin - if not isinstance(self.data_source, tables.Table): + if not isinstance(self.leaf, tables.Table): # Leaf is some kind of PyTables array - atom_type = self.data_source.atom.type + atom_type = self.leaf.atom.type if atom_type == 'object': self.formatContent = vitables.utils.formatObjectContent elif atom_type in ('vlstring', 'vlunicode'): @@ -143,8 +143,8 @@ def headerData(self, section, orientation, role): if orientation == QtCore.Qt.Horizontal: # For tables horizontal labels are column names, for arrays # the section numbers are used as horizontal labels - if hasattr(self.data_source, 'description'): - return str(self.data_source.colnames[section]) + if hasattr(self.leaf, 'description'): + return str(self.leaf.colnames[section]) return str(section) ## Rows-labels From 63f39f0238898cda09ae179a7a074076bb9c0038 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Wed, 14 Jun 2017 01:24:25 +0200 Subject: [PATCH 06/13] feat(table): drop minor forgotten enum_indices method --- vitables/vttables/leaf_model.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/vitables/vttables/leaf_model.py b/vitables/vttables/leaf_model.py index 8c599be1..20cdf406 100644 --- a/vitables/vttables/leaf_model.py +++ b/vitables/vttables/leaf_model.py @@ -114,9 +114,6 @@ def __init__(self, rbuffer, parent=None): super(LeafModel, self).__init__(parent) - def _collect_enum_indices(self): - """Initialize structures required to properly display enum columns.""" - def headerData(self, section, orientation, role): """Returns the data for the given role and section in the header with the specified orientation. From 7c8ca9f922066ad46427f52338a84d3b2b2c169b Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Wed, 14 Jun 2017 02:02:46 +0200 Subject: [PATCH 07/13] refact(scroll): move slider max-value calculation in scrollbar-class --- vitables/vttables/leaf_view.py | 89 ++++++++++------------------------ vitables/vttables/scrollbar.py | 27 ++++++----- 2 files changed, 42 insertions(+), 74 deletions(-) diff --git a/vitables/vttables/leaf_view.py b/vitables/vttables/leaf_view.py index 419e849d..88844d05 100644 --- a/vitables/vttables/leaf_view.py +++ b/vitables/vttables/leaf_view.py @@ -43,6 +43,7 @@ import vitables.vttables.scrollbar as scrollbar import vitables.vttables.leaf_delegate as leaf_delegate + class LeafView(QtWidgets.QTableView): """ A view for real data contained in leaves. @@ -79,8 +80,8 @@ def __init__(self, tmodel, parent=None): self.rbuffer_fault = False self.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOn) self.tricky_vscrollbar = scrollbar.ScrollBar(self) - self.max_value = self.tvsMaxValue() - self.tricky_vscrollbar.setMaximum(self.max_value) + self.max_value = self.tricky_vscrollbar.setMaxValue( + self.leaf_numrows) self.tricky_vscrollbar.setMinimum(0) self.interval_size = self.mapSlider2Leaf() @@ -109,20 +110,6 @@ def __init__(self, tmodel, parent=None): self.tricky_vscrollbar.actionTriggered.connect( self.navigateWithMouse) - - def tvsMaxValue(self): - """Calulate the maximum range value of the tricky vertical scrollbar. - """ - - # The scrollbar range must be a signed 32 bits integer so its - # largest range is [0, 2**31 - 1] - top_value = 2**31 - 1 - if self.leaf_numrows <= top_value: - top_value = self.leaf_numrows - return top_value - - - def mapSlider2Leaf(self): """Setup the interval size. @@ -147,7 +134,6 @@ def mapSlider2Leaf(self): interval_size = round(self.leaf_numrows / self.max_value) return interval_size - def syncView(self): """Update the tricky scrollbar value after a data navigation. @@ -160,15 +146,14 @@ def syncView(self): fv_label = self.vheader.logicalIndexAt(0) + offset lv_label = self.vheader.logicalIndexAt( self.vheader.viewport().height() - 1) + offset - if lv_label == self.leaf_numrows : + if lv_label == self.leaf_numrows: self.tricky_vscrollbar.setValue(self.max_value) - elif fv_label == 1 : + elif fv_label == 1: self.tricky_vscrollbar.setValue(0) - else : + else: value = round(fv_label / self.interval_size) self.tricky_vscrollbar.setValue(value) - def updateView(self): """Update the view contents after a buffer fault. """ @@ -177,10 +162,9 @@ def updateView(self): QtCore.Qt.Vertical, 0, self.tmodel.numrows - 1) top_left = self.tmodel.index(0, 0) bottom_right = self.tmodel.index(self.tmodel.numrows - 1, - self.tmodel.numcols - 1) + self.tmodel.numcols - 1) self.dataChanged(top_left, bottom_right) - def navigateWithMouse(self, slider_action): """Navigate the view with the mouse. @@ -213,7 +197,7 @@ def navigateWithMouse(self, slider_action): 3: self.addPageStep, 4: self.subPageStep, 7: self.dragSlider - } + } if not slider_action in actions.keys(): return # Navigate the data dealing with buffer faults @@ -224,7 +208,6 @@ def navigateWithMouse(self, slider_action): # reference self.syncView() - def mouseNavInfo(self, direction): """Gives information about model, vertical header and viewport. @@ -249,7 +232,6 @@ def mouseNavInfo(self, direction): return (model, vh, table_rows, buffer_start, row, page_step) - def addSingleStep(self): """Setup data for moving towards the last section line by line. """ @@ -260,7 +242,7 @@ def addSingleStep(self): # row of the dataset we still can go downwards so we have to # read the next contiguous buffer if (last_vp_row + 1 == table_rows) and \ - (buffer_start + table_rows < self.leaf_numrows): + (buffer_start + table_rows < self.leaf_numrows): # Buffer fault. The new buffer starts just after the current # first row of the viewport. new_start = buffer_start + last_vp_row - page_step + 1 @@ -272,7 +254,6 @@ def addSingleStep(self): else: self.vscrollbar.triggerAction(1) - def addPageStep(self): """Setup data for moving towards the last section page by page. """ @@ -283,7 +264,7 @@ def addPageStep(self): # row of the dataset we still can go downwards so we have to # read the next contiguous buffer if (last_vp_row + page_step + 1 > table_rows) and \ - (buffer_start + table_rows < self.leaf_numrows): + (buffer_start + table_rows < self.leaf_numrows): # Buffer fault. The new buffer starts at the current last # row of the viewport. new_start = buffer_start + last_vp_row @@ -295,7 +276,6 @@ def addPageStep(self): else: self.vscrollbar.triggerAction(3) - def subSingleStep(self): """Setup data for moving towards the first section line by line. """ @@ -313,12 +293,11 @@ def subSingleStep(self): self.scrollTo( model.index( buffer_start + page_step - model.rbuffer.start - 1, 0), - QtWidgets.QAbstractItemView.PositionAtBottom) + QtWidgets.QAbstractItemView.PositionAtBottom) self.updateView() else: self.vscrollbar.triggerAction(2) - def subPageStep(self): """Setup data for moving towards the first section page by page. """ @@ -332,8 +311,8 @@ def subPageStep(self): # Buffer fault. The new buffer ends just at the current # first row of the viewport. model.loadData( - buffer_start + first_vp_row - table_rows + 1, - table_rows) + buffer_start + first_vp_row - table_rows + 1, + table_rows) self.updateView() self.scrollTo( model.index( @@ -342,7 +321,6 @@ def subPageStep(self): else: self.vscrollbar.triggerAction(4) - def dragSlider(self): """Move the slider by dragging it. @@ -369,7 +347,7 @@ def dragSlider(self): elif value >= self.max_value: value = self.max_value row = self.leaf_numrows - 1 - else : + else: row = self.interval_size * value # top buffer fault condition @@ -387,12 +365,11 @@ def dragSlider(self): self.vscrollbar.triggerAction( QtWidgets.QAbstractSlider.SliderToMaximum) # we are somewhere in the middle of the dataset - else : + else: self.scrollTo( model.index(row - model.rbuffer.start, 0), QtWidgets.QAbstractItemView.PositionAtTop) - def topBF(self, value, row): """Going out of buffer when browsing upwards. @@ -420,7 +397,6 @@ def topBF(self, value, row): self.updateView() self.scrollTo(self.tmodel.index(position, 0), hint) - def bottomBF(self, value, row): """Going out of buffer when browsing downwards. @@ -449,7 +425,6 @@ def bottomBF(self, value, row): self.updateView() self.scrollTo(self.tmodel.index(position, 0), hint) - def wheelEvent(self, event): """Specialized handler for the wheel events received by the *viewport*. @@ -474,7 +449,6 @@ def wheelEvent(self, event): else: QtWidgets.QTableView.wheelEvent(self, event) - def wheelDown(self, event): """Setup data for wheeling with the mouse towards the last section. """ @@ -494,12 +468,11 @@ def wheelDown(self, event): model.loadData(new_start, table_rows) self.updateView() self.scrollTo( - model.index(new_start - model.rbuffer.start, 0), - QtWidgets.QAbstractItemView.PositionAtTop) + model.index(new_start - model.rbuffer.start, 0), + QtWidgets.QAbstractItemView.PositionAtTop) else: QtCore.QCoreApplication.sendEvent(self.vscrollbar, event) - def wheelUp(self, event): """Setup data for wheeling with the mouse towards the first section. """ @@ -524,7 +497,6 @@ def wheelUp(self, event): else: QtCore.QCoreApplication.sendEvent(self.vscrollbar, event) - def keyPressEvent(self, event): """Handle basic cursor movement for key events. @@ -556,7 +528,6 @@ def keyPressEvent(self, event): else: QtWidgets.QTableView.keyPressEvent(self, event) - def homeKeyPressEvent(self): """Specialised handler for the `Home` key press event. @@ -567,7 +538,7 @@ def homeKeyPressEvent(self): table_rows = model.numrows index = model.index(0, 0) # Update buffer if needed - if model.rbuffer.start > 0 : + if model.rbuffer.start > 0: model.loadData(0, table_rows) self.updateView() self.setCurrentIndex(index) @@ -577,7 +548,6 @@ def homeKeyPressEvent(self): # the displayed data self.tricky_vscrollbar.setValue(0) - def endKeyPressEvent(self): """Specialised handler for the `End` key press event. @@ -591,7 +561,7 @@ def endKeyPressEvent(self): last_row = model.rbuffer.start + table_rows if last_row < self.leaf_numrows: self.tmodel.loadData(self.leaf_numrows - table_rows, - table_rows) + table_rows) self.updateView() self.setCurrentIndex(index) self.scrollToBottom() @@ -600,7 +570,6 @@ def endKeyPressEvent(self): # the displayed data self.tricky_vscrollbar.setValue(self.max_value) - def keyboardNavInfo(self): """Gives information about model, and current cell. @@ -627,8 +596,7 @@ def keyboardNavInfo(self): dataset_row = buffer_start + buffer_row return (model, table_rows, buffer_start, page_step, current_index, - buffer_row, buffer_column, dataset_row) - + buffer_row, buffer_column, dataset_row) def upKeyPressEvent(self, event): """Specialised handler for the cursor up key press event. @@ -652,7 +620,7 @@ def upKeyPressEvent(self, event): index = model.index(row, buffer_column) self.setCurrentIndex(index) self.scrollTo(index, - QtWidgets.QAbstractItemView.PositionAtTop) + QtWidgets.QAbstractItemView.PositionAtTop) else: QtWidgets.QTableView.keyPressEvent(self, event) @@ -661,7 +629,6 @@ def upKeyPressEvent(self, event): # reference self.syncView() - def pageUpKeyPressEvent(self, event): """Specialised handler for the `PageUp` key press event. @@ -684,7 +651,7 @@ def pageUpKeyPressEvent(self, event): index = model.index(row, buffer_column) self.setCurrentIndex(index) self.scrollTo(index, - QtWidgets.QAbstractItemView.PositionAtTop) + QtWidgets.QAbstractItemView.PositionAtTop) else: QtWidgets.QTableView.keyPressEvent(self, event) @@ -693,7 +660,6 @@ def pageUpKeyPressEvent(self, event): # reference self.syncView() - def downKeyPressEvent(self, event): """Specialised handler for the cursor down key press event. @@ -717,7 +683,7 @@ def downKeyPressEvent(self, event): index = model.index(row, buffer_column) self.setCurrentIndex(index) self.scrollTo(index, - QtWidgets.QAbstractItemView.PositionAtBottom) + QtWidgets.QAbstractItemView.PositionAtBottom) else: QtWidgets.QTableView.keyPressEvent(self, event) @@ -726,7 +692,6 @@ def downKeyPressEvent(self, event): # reference self.syncView() - def pageDownKeyPressEvent(self, event): """Specialised handler for the `PageDown` key press event. @@ -750,7 +715,7 @@ def pageDownKeyPressEvent(self, event): index = model.index(row, buffer_column) self.setCurrentIndex(index) self.scrollTo(index, - QtWidgets.QAbstractItemView.PositionAtBottom) + QtWidgets.QAbstractItemView.PositionAtBottom) else: QtWidgets.QTableView.keyPressEvent(self, event) @@ -787,7 +752,6 @@ def currentChanged(self, current, previous): if self.tmodel.numrows < self.leaf_numrows: self.valid_current_buffer = self.tmodel.rbuffer.start - # This method has been renamed from loadDatasetCurrentCell to # validCurrentCellBuffer. The method has been debugged too def validCurrentCellBuffer(self): @@ -797,12 +761,11 @@ def validCurrentCellBuffer(self): table_rows = self.tmodel.numrows valid_current = self.currentIndex().row() + self.valid_current_buffer if not (self.tmodel.rbuffer.start <= - valid_current <= - self.tmodel.rbuffer.start + table_rows - 1): + valid_current <= + self.tmodel.rbuffer.start + table_rows - 1): self.tmodel.loadData(self.valid_current_buffer, table_rows) self.updateView() - def selectionChanged(self, selected, deselected): """Track the dataset selected cells. diff --git a/vitables/vttables/scrollbar.py b/vitables/vttables/scrollbar.py index 583ef23e..0d3d7da3 100644 --- a/vitables/vttables/scrollbar.py +++ b/vitables/vttables/scrollbar.py @@ -33,11 +33,12 @@ __docformat__ = 'restructuredtext' -from qtpy import QtCore -from qtpy import QtGui -from qtpy import QtWidgets +from qtpy.QtCore import Qt +from qtpy.QtCore import QEvent +from qtpy.QtWidgets import QScrollBar -class ScrollBar(QtWidgets.QScrollBar): + +class ScrollBar(QScrollBar): """ A specialised scrollbar for views of huge datasets. @@ -60,15 +61,19 @@ def __init__(self, view): super(ScrollBar, self).__init__(parent) view.vscrollbar.setVisible(False) parent.layout().addWidget(self) - self.setOrientation(QtCore.Qt.Vertical) + self.setOrientation(Qt.Vertical) self.setObjectName('tricky_vscrollbar') - def event(self, e): - """Filter wheel events and send them to the table viewport. - """ - - if (e.type() == QtCore.QEvent.Wheel): + """Filter wheel events and send them to the table viewport. """ + if (e.type() == QEvent.Wheel): self.view.wheelEvent(e) return True - return QtWidgets.QScrollBar.event(self, e) + return QScrollBar.event(self, e) + + def setMaxValue(self, max_value): + """Ensure range of scrollbar is a signed 32bit integer.""" + max_value = min(2 ** 31 - 1, max_value) + self.setMaximum(max_value) + + return max_value From 2ef09d4a7eea0aff0f2c0f7da11e8e72ffa9c9d2 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Tue, 13 Jun 2017 22:18:08 +0200 Subject: [PATCH 08/13] refact(buffer): MOVE PAGING from buffer-->model to DECOUPLE view + Move start/nrows/chunk_size attributes up, from buffer-->model. + Drop or move buffer-fields, to decouple it from model & view (ie. to facilitate replacing it with a DataFrame). + Move & merge buffer.getReadParameters()-->model.loadData(). + DROP `model.chunk_size`, it is identical to `nrows`. - Indexing errors in `buffer.getCell()` don't report `start` anymore :-( --- vitables/plugins/timeseries/time_series.py | 10 +- vitables/vttables/buffer.py | 82 ++++---------- vitables/vttables/datasheet.py | 9 +- vitables/vttables/leaf_delegate.py | 2 +- vitables/vttables/leaf_model.py | 126 +++++++++++---------- vitables/vttables/leaf_view.py | 66 +++++------ 6 files changed, 127 insertions(+), 168 deletions(-) diff --git a/vitables/plugins/timeseries/time_series.py b/vitables/plugins/timeseries/time_series.py index f07219db..183b1a04 100644 --- a/vitables/plugins/timeseries/time_series.py +++ b/vitables/plugins/timeseries/time_series.py @@ -326,18 +326,18 @@ def table_data(self, index, role=QtCore.Qt.DisplayRole): - `role`: the role being returned """ - if not index.isValid() or not (0 <= index.row() < self.numrows): + if not index.isValid() or not (0 <= index.row() < self.nrows): return None - + if role == QtCore.Qt.DisplayRole: cell = self.rbuffer.getCell(index.row(), index.column()) if index.column() in self.ts_cols: return self.tsFormatter(cell) return self.formatContent(cell) - + if role == QtCore.Qt.TextAlignmentRole: return QtCore.Qt.AlignLeft | QtCore.Qt.AlignTop - + return None @@ -353,7 +353,7 @@ def array_data(self, index, role=QtCore.Qt.DisplayRole): - `role`: the role being returned """ - if not index.isValid() or not (0 <= index.row() < self.numrows): + if not index.isValid() or not (0 <= index.row() < self.nrows): return None if role == QtCore.Qt.DisplayRole: diff --git a/vitables/vttables/buffer.py b/vitables/vttables/buffer.py index 90013b42..e9ae953c 100644 --- a/vitables/vttables/buffer.py +++ b/vitables/vttables/buffer.py @@ -30,17 +30,14 @@ __docformat__ = 'restructuredtext' -import warnings import logging - -from qtpy import QtGui -from qtpy import QtWidgets +import vitables.utils +import warnings import numpy +from qtpy import QtWidgets import tables -import vitables.utils - translate = QtWidgets.QApplication.translate # Restrict the available flavors to 'numpy' so that reading a leaf @@ -89,17 +86,8 @@ def __init__(self, leaf): self.leaf = leaf - # The maximum number of rows to be read from the data source - self.chunk_size = 10000 - - # The length of the dimension that is going to be read. - self.leaf_numrows = self.leafNumberOfRows() - if self.leaf_numrows <= self.chunk_size: - self.chunk_size = self.leaf_numrows - # The numpy array where read data will be stored self.chunk = numpy.array([]) - self.start = 0 # The method used for reading data depends on the kind of node. # Setting the reader method at initialization time increases the @@ -126,10 +114,11 @@ def __init__(self, leaf): def __del__(self): """Release resources before destroying the buffer. """ + # FIXME: PY3.5+ leaks resources (use finalizer instead). self.chunk = None - def leafNumberOfRows(self): - """The number of rows of the dataset being read. + def total_nrows(self): + """Estimates the number of rows of the dataset being read. We don't use the `Leaf.nrows` attribute because it is not always suitable for displaying the data in a 2D grid. Instead we use the @@ -158,37 +147,6 @@ def leafNumberOfRows(self): return nrows - def getReadParameters(self, start, buffer_size): - """ - Returns acceptable parameters for the read method. - - :Parameters: - - - `start`: the document row that is the first row of the chunk. - - `buffer_size`: the buffer size, i.e. the number of rows to be read. - - :Returns: - a tuple with tested values for the parameters of the read method - """ - - first_row = 0 - last_row = self.leaf_numrows - - # When scrolling up we must keep start value >= first_row - if start < first_row: - start = first_row - - # When scrolling down we must keep stop value <= last_row - stop = start + buffer_size - if stop > last_row: - stop = last_row - - # Ensure that the whole buffer will be filled - if stop - start < self.chunk_size: - start = stop - self.chunk_size - - return start, stop - def isDataSourceReadable(self): """Find out if the dataset can be read or not. @@ -202,7 +160,7 @@ def isDataSourceReadable(self): self.leaf.read(0, 1) except tables.HDF5ExtError: readable = False - ## TODO: Fix error msg to include exception. + # TODO: Fix error msg to include exception or merge with below. self.logger.error( translate('Buffer', """Problems reading records. The dataset """ @@ -218,7 +176,7 @@ def isDataSourceReadable(self): 'A dataset read error').format(e.message)) return readable - def readBuffer(self, start, buffer_size): + def readBuffer(self, start, stop): """ Read a chunk from the data source. @@ -233,11 +191,10 @@ def readBuffer(self, start, buffer_size): :Parameters: - - `start`: the document row that is the first row of the chunk. - - `buffer_size`: the buffer size, i.e. the number of rows to be read. + :param start: the document row that is the first row of the chunk. + :param stop: the last row to read, inclusive. """ - start, stop = self.getReadParameters(start, buffer_size) try: # data_source is a tables.Table or a tables.XArray # but data is a numpy array @@ -246,7 +203,7 @@ def readBuffer(self, start, buffer_size): # array returned by EArray.read() will have only 2 rows data = self.leaf.read(start, stop) except tables.HDF5ExtError: - ## TODO: Fix error msg to include exception. + # TODO: Fix error msg to include exception. self.logger.error( translate('Buffer', """\nError: problems reading records. """ """The dataset maybe corrupted.""", @@ -256,7 +213,6 @@ def readBuffer(self, start, buffer_size): else: # Update the buffer contents and its start position self.chunk = data - self.start = start def scalarCell(self, row, col): """ @@ -276,8 +232,8 @@ def scalarCell(self, row, col): try: return self.chunk[()] except IndexError: - self.logger.error('IndexError! buffer start: {0} row, column: ' - '{1}, {2}'.format(self.start, row, col)) + self.logger.error('IndexError! row, column: {1}, {2}' + .format(row, col)) def vectorCell(self, row, col): """ @@ -307,8 +263,8 @@ def vectorCell(self, row, col): try: return self.chunk[row] except IndexError: - self.logger.error('IndexError! buffer start: {0} row, column: ' - '{1}, {2}'.format(self.start, row, col)) + self.logger.error('IndexError! row, column: {1}, {2}' + .format(row, col)) def EArrayCell(self, row, col): """ @@ -334,8 +290,8 @@ def EArrayCell(self, row, col): try: return self.leaf.read()[row] except IndexError: - self.logger.error('IndexError! buffer start: {0} row, column: ' - '{1}, {2}'.format(self.start, row, col)) + self.logger.error('IndexError! row, column: {1}, {2}' + .format(row, col)) def arrayCell(self, row, col): """ @@ -363,5 +319,5 @@ def arrayCell(self, row, col): try: return self.chunk[row][col] except IndexError: - self.logger.error('IndexError! buffer start: {0} row, column: ' - '{1}, {2}'.format(self.start, row, col)) + self.logger.error('IndexError! row, column: {1}, {2}' + .format(row, col)) diff --git a/vitables/vttables/datasheet.py b/vitables/vttables/datasheet.py index 833bbb98..eb94e40c 100644 --- a/vitables/vttables/datasheet.py +++ b/vitables/vttables/datasheet.py @@ -37,7 +37,7 @@ import vitables.vtwidgets.zoom_cell as zoom_cell import vitables.vttables.leaf_model as leaf_model import vitables.vttables.leaf_view as leaf_view -import vitables.vttables.buffer as readBuffer + class DataSheet(QtWidgets.QMdiSubWindow): """ @@ -67,8 +67,7 @@ def __init__(self, index): else: leaf = pt_node - rbuffer = readBuffer.Buffer(leaf) - self.leaf_model = leaf_model.LeafModel(rbuffer) + self.leaf_model = leaf_model.LeafModel(leaf) self.leaf_view = leaf_view.LeafView(self.leaf_model) super(DataSheet, self).__init__(self.vtgui.workspace, @@ -162,10 +161,10 @@ def zoomCell(self, index): if node.node_kind == 'table': col = info.columns_names[column] title = '{0}: {1}[{2}]'.format(node.name, col, - tmodel.rbuffer.start + row) + tmodel.start + row) else: title = '{0}: ({1},{2})'.format(node.name, - tmodel.rbuffer.start + row, column) + tmodel.start + row, column) zoom_cell.ZoomCell(data, title, self.vtgui.workspace, self.dbt_leaf) diff --git a/vitables/vttables/leaf_delegate.py b/vitables/vttables/leaf_delegate.py index 5fb102ed..5e757b1b 100644 --- a/vitables/vttables/leaf_delegate.py +++ b/vitables/vttables/leaf_delegate.py @@ -67,7 +67,7 @@ def paint(self, painter, option, index): # option.state is an ORed combination of flags if (option.state & QtWidgets.QStyle.State_Selected): model = index.model() - buffer_start = model.rbuffer.start + buffer_start = model.start cell = index.model().selected_cell if ((index == cell['index']) and \ (buffer_start != cell['buffer_start'])): diff --git a/vitables/vttables/leaf_model.py b/vitables/vttables/leaf_model.py index 20cdf406..b0cd5319 100644 --- a/vitables/vttables/leaf_model.py +++ b/vitables/vttables/leaf_model.py @@ -26,14 +26,16 @@ __docformat__ = 'restructuredtext' -import collections +import vitables.utils +from qtpy import QtCore import tables -import numpy as np -from qtpy import QtCore +from vitables.vttables import buffer -import vitables.utils + +#: The maximum number of rows to be read from the data source. +CHUNK_SIZE = 10000 class LeafModel(QtCore.QAbstractTableModel): @@ -42,46 +44,50 @@ class LeafModel(QtCore.QAbstractTableModel): The data is read from data sources (i.e., `HDF5/PyTables` nodes) by the model. + The dataset number of rows is potentially huge but tables are read and + displayed in chunks. + + + :param parent: + The parent of the model, passed as is in the superclass. + :attribute leaf: + the underlying hdf5 data + :attribute rbuffer: + Code for chunking and inspecting the undelying data. + :attribute leaf_numrows: + the total number of rows in the underlying data + :attribute numrows: + The number of rows visible which equals the chunking-size. + :attribute numcols: + The total number of columnss visible, equal to those visible. + :attribute start: + The zero-based starting index of the chunk within the total rows. - :Parameters: - - - `rbuffer`: a buffer used for optimizing read access to data - - `parent`: the parent of the model """ - def __init__(self, rbuffer, parent=None): + def __init__(self, leaf, parent=None): """Create the model. """ # The model data source (a PyTables/HDF5 leaf) and its access buffer - self.leaf = rbuffer.leaf - self.rbuffer = rbuffer - - # The number of digits of the last row - self.last_row_width = len(str(self.rbuffer.leaf_numrows)) - - # - # The table dimensions - # + self.leaf = leaf + self.rbuffer = buffer.Buffer(leaf) - # The dataset number of rows is potentially huge but tables are - # kept small: just the data returned by a read operation of the - # buffer are displayed - self.numrows = self.rbuffer.leafNumberOfRows() - if self.numrows > self.rbuffer.chunk_size: - self.numrows = self.rbuffer.chunk_size + self.leaf_numrows = self.rbuffer.total_nrows() + self.numrows = min(self.leaf_numrows, CHUNK_SIZE) + self.start = 0 # The dataset number of columns doesn't use to be large so, we don't # need set a maximum as we did with rows. The whole set of columns # are displayed - if isinstance(self.leaf, tables.Table): + if isinstance(leaf, tables.Table): # Leaf is a PyTables table - self.numcols = len(self.leaf.colnames) - elif isinstance(self.leaf, tables.EArray): + self.numcols = len(leaf.colnames) + elif isinstance(leaf, tables.EArray): self.numcols = 1 else: # Leaf is some kind of PyTables array - shape = self.leaf.shape + shape = leaf.shape if len(shape) > 1: # The leaf will be displayed as a bidimensional matrix self.numcols = shape[1] @@ -98,9 +104,9 @@ def __init__(self, rbuffer, parent=None): # Time series (if they are found) are formatted transparently # via the time_series.py plugin - if not isinstance(self.leaf, tables.Table): + if not isinstance(leaf, tables.Table): # Leaf is some kind of PyTables array - atom_type = self.leaf.atom.type + atom_type = leaf.atom.type if atom_type == 'object': self.formatContent = vitables.utils.formatObjectContent elif atom_type in ('vlstring', 'vlunicode'): @@ -110,7 +116,7 @@ def __init__(self, rbuffer, parent=None): self.selected_cell = {'index': QtCore.QModelIndex(), 'buffer_start': 0} # Populate the model with the first chunk of data - self.loadData(self.rbuffer.start, self.rbuffer.chunk_size) + self.loadData(0, self.numrows) super(LeafModel, self).__init__(parent) @@ -136,7 +142,7 @@ def headerData(self, section, orientation, role): if role != QtCore.Qt.DisplayRole: return None - ## Columns-labels + # Columns-labels if orientation == QtCore.Qt.Horizontal: # For tables horizontal labels are column names, for arrays # the section numbers are used as horizontal labels @@ -144,8 +150,8 @@ def headerData(self, section, orientation, role): return str(self.leaf.colnames[section]) return str(section) - ## Rows-labels - return str(self.rbuffer.start + section) + # Rows-labels + return str(self.start + section) def data(self, index, role=QtCore.Qt.DisplayRole): """Returns the data stored under the given role for the item @@ -174,43 +180,45 @@ def data(self, index, role=QtCore.Qt.DisplayRole): def columnCount(self, index=QtCore.QModelIndex()): """The number of columns of the given model index. - When implementing a table based model this method has to be overriden - -because it is an abstract method- and should return 0 for valid - indices (because they have no children). If the index is not valid the - method should return the number of columns exposed by the model. + Overridden to return 0 for valid indices because they have no children; + otherwise return the total number of *columns* exposed by the model. - :Parameter index: the model index being inspected. + :param index: + the model index being inspected. """ - if not index.isValid(): - return self.numcols - else: - return 0 - + return 0 if index.isValid() else self.numcols def rowCount(self, index=QtCore.QModelIndex()): """The number of columns for the children of the given index. - When implementing a table based model this method has to be overriden - -because it is an abstract method- and should return 0 for valid - indices (because they have no children). If the index is not valid the - method should return the number of rows exposed by the model. + Overridden to return 0 for valid indices because they have no children; + otherwise return the total number of *rows* exposed by the model. :Parameter index: the model index being inspected. """ - if not index.isValid(): - return self.numrows - else: - return 0 + return 0 if index.isValid() else self.numrows + + def loadData(self, start, length): + """Load the model with fresh chunk from the underlying leaf. + :param start: + The first row (within the total nrows) of the chunk to read. + :param length: + The buffer size, i.e. the number of rows to be read. + """ - def loadData(self, start, chunk_size): - """Load the model with fresh data from the buffer. + # Enforce scrolling limits. + # + start = max(start, 0) + stop = min(start + length, self.leaf_numrows) + assert stop >= start, (self.numrows, start, stop, length) - :Parameters: + # Ensure buffer filled when scrolled beyond bottom. + # + actual_start = stop - self.numrows + start = max(min(actual_start, start), 0) - - `start`: the row where the buffer starts - - `chunk_size`: the size of the buffer - """ - self.rbuffer.readBuffer(start, chunk_size) + self.rbuffer.readBuffer(start, stop) + self.start = start diff --git a/vitables/vttables/leaf_view.py b/vitables/vttables/leaf_view.py index 88844d05..b842feb6 100644 --- a/vitables/vttables/leaf_view.py +++ b/vitables/vttables/leaf_view.py @@ -34,8 +34,6 @@ __docformat__ = 'restructuredtext' -import numpy - from qtpy import QtCore from qtpy import QtGui from qtpy import QtWidgets @@ -63,7 +61,7 @@ def __init__(self, tmodel, parent=None): super(LeafView, self).__init__(parent) self.setAttribute(QtCore.Qt.WA_DeleteOnClose) self.tmodel = tmodel # This is a MUST - self.leaf_numrows = self.tmodel.rbuffer.leaf_numrows + self.leaf_numrows = leaf_numrows = self.tmodel.leaf_numrows self.selection_model = self.selectionModel() self.setSelectionMode(QtWidgets.QAbstractItemView.SingleSelection) self.setSelectionBehavior(QtWidgets.QAbstractItemView.SelectItems) @@ -75,7 +73,7 @@ def __init__(self, tmodel, parent=None): self.setModel(tmodel) # For potentially huge datasets use a customised scrollbar - if self.leaf_numrows > self.tmodel.numrows: + if leaf_numrows > tmodel.numrows: self.setItemDelegate(leaf_delegate.LeafDelegate()) self.rbuffer_fault = False self.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOn) @@ -91,22 +89,22 @@ def __init__(self, tmodel, parent=None): font = self.vheader.font() font.setBold(True) fmetrics = QtGui.QFontMetrics(font) - max_width = fmetrics.width(" {0} ".format(str(self.leaf_numrows))) + max_width = fmetrics.width(" {0} ".format(str(leaf_numrows))) self.vheader.setMinimumWidth(max_width) self.vheader.setSectionsClickable(True) # Setup the headers' resize mode rmode = QtWidgets.QHeaderView.Stretch - if self.tmodel.columnCount() == 1: + if tmodel.columnCount() == 1: self.horizontalHeader().setSectionResizeMode(rmode) - if self.tmodel.rowCount() == 1: + if tmodel.rowCount() == 1: self.vheader.setSectionResizeMode(rmode) # Setup the text elide mode self.setTextElideMode(QtCore.Qt.ElideRight) # Connect signals to slots - if self.leaf_numrows > self.tmodel.numrows: + if leaf_numrows > tmodel.numrows: self.tricky_vscrollbar.actionTriggered.connect( self.navigateWithMouse) @@ -142,7 +140,7 @@ def syncView(self): update is done using the first visible row as a reference. """ - offset = self.tmodel.rbuffer.start + 1 + offset = self.tmodel.start + 1 fv_label = self.vheader.logicalIndexAt(0) + offset lv_label = self.vheader.logicalIndexAt( self.vheader.viewport().height() - 1) + offset @@ -221,7 +219,7 @@ def mouseNavInfo(self, direction): # About the table table_rows = model.numrows - buffer_start = model.rbuffer.start + buffer_start = model.start # The viewport BEFORE navigating the data if (direction == 'u'): @@ -249,7 +247,7 @@ def addSingleStep(self): model.loadData(new_start, table_rows) self.updateView() self.scrollTo( - model.index(new_start - model.rbuffer.start, 0), + model.index(new_start - model.start, 0), QtWidgets.QAbstractItemView.PositionAtTop) else: self.vscrollbar.triggerAction(1) @@ -271,7 +269,7 @@ def addPageStep(self): model.loadData(new_start, table_rows) self.updateView() self.scrollTo( - model.index(new_start - model.rbuffer.start, 0), + model.index(new_start - model.start, 0), QtWidgets.QAbstractItemView.PositionAtTop) else: self.vscrollbar.triggerAction(3) @@ -291,8 +289,7 @@ def subSingleStep(self): model.loadData( buffer_start + page_step - table_rows, table_rows) self.scrollTo( - model.index( - buffer_start + page_step - model.rbuffer.start - 1, 0), + model.index(buffer_start + page_step - model.start - 1, 0), QtWidgets.QAbstractItemView.PositionAtBottom) self.updateView() else: @@ -316,7 +313,7 @@ def subPageStep(self): self.updateView() self.scrollTo( model.index( - buffer_start + first_vp_row - model.rbuffer.start, 0), + buffer_start + first_vp_row - model.start, 0), QtWidgets.QAbstractItemView.PositionAtBottom) else: self.vscrollbar.triggerAction(4) @@ -351,10 +348,10 @@ def dragSlider(self): row = self.interval_size * value # top buffer fault condition - if row < model.rbuffer.start: + if row < model.start: self.topBF(value, row) # bottom buffer fault condition - elif (row >= model.rbuffer.start + table_rows): + elif (row >= model.start + table_rows): self.bottomBF(value, row) # We are at top of the dataset elif value == self.tricky_vscrollbar.minimum(): @@ -367,13 +364,13 @@ def dragSlider(self): # we are somewhere in the middle of the dataset else: self.scrollTo( - model.index(row - model.rbuffer.start, 0), + model.index(row - model.start, 0), QtWidgets.QAbstractItemView.PositionAtTop) def topBF(self, value, row): """Going out of buffer when browsing upwards. - Buffer fault condition: row < rbuffer.start + Buffer fault condition: row < model.start :Parameters: @@ -400,7 +397,7 @@ def topBF(self, value, row): def bottomBF(self, value, row): """Going out of buffer when browsing downwards. - Buffer fault condition: row > self.tmodel.rbuffer.start + table_rows - 1 + Buffer fault condition: row > self.tmodel.start + table_rows - 1 :Parameters: @@ -467,9 +464,8 @@ def wheelDown(self, event): buffer_start + last_vp_row + self.wheel_step - page_step model.loadData(new_start, table_rows) self.updateView() - self.scrollTo( - model.index(new_start - model.rbuffer.start, 0), - QtWidgets.QAbstractItemView.PositionAtTop) + self.scrollTo(model.index(new_start - model.start, 0), + QtWidgets.QAbstractItemView.PositionAtTop) else: QtCore.QCoreApplication.sendEvent(self.vscrollbar, event) @@ -492,7 +488,7 @@ def wheelUp(self, event): self.updateView() self.scrollTo( model.index( - new_start + table_rows - model.rbuffer.start - 1, 0), + new_start + table_rows - model.start - 1, 0), QtWidgets.QAbstractItemView.PositionAtBottom) else: QtCore.QCoreApplication.sendEvent(self.vscrollbar, event) @@ -538,7 +534,7 @@ def homeKeyPressEvent(self): table_rows = model.numrows index = model.index(0, 0) # Update buffer if needed - if model.rbuffer.start > 0: + if model.start > 0: model.loadData(0, table_rows) self.updateView() self.setCurrentIndex(index) @@ -558,7 +554,7 @@ def endKeyPressEvent(self): table_rows = model.numrows index = model.index(table_rows - 1, model.numcols - 1) # Update buffer if needed - last_row = model.rbuffer.start + table_rows + last_row = model.start + table_rows if last_row < self.leaf_numrows: self.tmodel.loadData(self.leaf_numrows - table_rows, table_rows) @@ -585,7 +581,7 @@ def keyboardNavInfo(self): # About the table table_rows = model.numrows - buffer_start = model.rbuffer.start + buffer_start = model.start page_step = self.vscrollbar.pageStep() @@ -614,7 +610,7 @@ def upKeyPressEvent(self, event): model.loadData(dataset_row - table_rows + page_step, table_rows) self.updateView() # The position of the new current row - row = dataset_row - model.rbuffer.start - 1 + row = dataset_row - model.start - 1 if row < 0: row = 0 index = model.index(row, buffer_column) @@ -645,7 +641,7 @@ def pageUpKeyPressEvent(self, event): model.loadData(dataset_row - table_rows, table_rows) self.updateView() # The position of the new current row - row = dataset_row - model.rbuffer.start - page_step - 1 + row = dataset_row - model.start - page_step - 1 if row < 0: row = 0 index = model.index(row, buffer_column) @@ -677,7 +673,7 @@ def downKeyPressEvent(self, event): model.loadData(dataset_row - page_step + 1, table_rows) self.updateView() # The position of the new current row - row = dataset_row - model.rbuffer.start + 1 + row = dataset_row - model.start + 1 if row > table_rows - 1: row = table_rows - 1 index = model.index(row, buffer_column) @@ -709,7 +705,7 @@ def pageDownKeyPressEvent(self, event): model.loadData(dataset_row + 1, table_rows) self.updateView() # The position of the new current row - row = dataset_row - model.rbuffer.start + page_step + 1 + row = dataset_row - model.start + page_step + 1 if row > table_rows - 1: row = table_rows - 1 index = model.index(row, buffer_column) @@ -750,7 +746,7 @@ def currentChanged(self, current, previous): QtWidgets.QTableView.currentChanged(self, current, previous) if self.tmodel.numrows < self.leaf_numrows: - self.valid_current_buffer = self.tmodel.rbuffer.start + self.valid_current_buffer = self.tmodel.start # This method has been renamed from loadDatasetCurrentCell to # validCurrentCellBuffer. The method has been debugged too @@ -760,9 +756,9 @@ def validCurrentCellBuffer(self): table_rows = self.tmodel.numrows valid_current = self.currentIndex().row() + self.valid_current_buffer - if not (self.tmodel.rbuffer.start <= + if not (self.tmodel.start <= valid_current <= - self.tmodel.rbuffer.start + table_rows - 1): + self.tmodel.start + table_rows - 1): self.tmodel.loadData(self.valid_current_buffer, table_rows) self.updateView() @@ -784,7 +780,7 @@ def selectionChanged(self, selected, deselected): if len(selection): model.selected_cell = { 'index': selection[0], - 'buffer_start': model.rbuffer.start, + 'buffer_start': model.start, } else: QtWidgets.QTableView.selectionChanged(self, selected, deselected) From 254264fc2e85af6794ffefcf1031e1c0fc06408f Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Wed, 14 Jun 2017 02:02:26 +0200 Subject: [PATCH 09/13] style(tabview): pep8 --- vitables/plugins/columnorg/columnar_org.py | 16 ++++++------- vitables/plugins/timeseries/time_series.py | 28 ++++++++-------------- vitables/vttables/leaf_view.py | 4 ++-- 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/vitables/plugins/columnorg/columnar_org.py b/vitables/plugins/columnorg/columnar_org.py index 71d8b7c3..ba13efca 100644 --- a/vitables/plugins/columnorg/columnar_org.py +++ b/vitables/plugins/columnorg/columnar_org.py @@ -146,7 +146,7 @@ def helpAbout(self, parent): 'folder': os.path.join(os.path.dirname(__file__)), 'author': 'Vicent Mas ', 'comment': translate('ArrayColsOrganizer', - """ + """

Plugin that provides an alternative view for arrays with the same number of rows.

The user has to open all the arrays she whishes to see in @@ -158,7 +158,7 @@ def helpAbout(self, parent):

Note that all ticked arrays must have the same dimensions for the `Group` menu item to enable. """, - 'Text of an About plugin message box')} + 'Text of an About plugin message box')} about_page = AboutPage(desc, parent) return about_page @@ -198,10 +198,10 @@ def addEntry(self): triggered=self.groupArrays, icon=object_group_icon, statusTip=translate('MenuUpdater', - """ + """ Use a unique widget to display Arrays as if they where columns of a Table""", - "Status bar text for the Node -> Group Arrays action")) + "Status bar text for the Node -> Group Arrays action")) object_ungroup_icon = QtGui.QIcon() pixmap = QtGui.QPixmap(os.path.join(_PLUGIN_FOLDER, @@ -219,8 +219,8 @@ def addEntry(self): triggered=self.ungroupArrays, icon=object_ungroup_icon, statusTip=translate('MenuUpdater', - """Ungroup previously grouped arrays.""", - "Status bar text for the Node -> Ungroup Arrays action")) + """Ungroup previously grouped arrays.""", + "Status bar text for the Node -> Ungroup Arrays action")) # Add the actions to the Node menu vitables.utils.addToMenu(self.vtgui.node_menu, (self.group_action, @@ -468,12 +468,12 @@ def customizeGroupedViews(self): nd = len(datasheets) for i in range(nd): - datasheets[nd-1].widget().verticalScrollBar().valueChanged.connect( + datasheets[nd - 1].widget().verticalScrollBar().valueChanged.connect( datasheets[i].widget().verticalScrollBar().setValue) if i < (nd - 1): datasheets[i].widget().verticalScrollBar().hide() datasheets[i].widget().setCornerWidget(None) - datasheets[i+1].widget().verticalHeader().hide() + datasheets[i + 1].widget().verticalHeader().hide() def closeEvent(self, event): """ Propagate the close event. diff --git a/vitables/plugins/timeseries/time_series.py b/vitables/plugins/timeseries/time_series.py index 183b1a04..6975e0e1 100644 --- a/vitables/plugins/timeseries/time_series.py +++ b/vitables/plugins/timeseries/time_series.py @@ -50,7 +50,6 @@ pd = None from qtpy import QtCore -from qtpy import QtGui from qtpy import QtWidgets import vitables.utils @@ -152,7 +151,7 @@ def tsFrequency(ts_kind, leaf): if ts_kind == 'scikits_ts': # The frequency of the time serie. Default is 6000 (daily) special_attrs = getattr(leaf._v_attrs, 'special_attrs', - {'freq': 6000}) + {'freq': 6000}) ts_freq = special_attrs['freq'] return ts_freq @@ -225,7 +224,7 @@ def customiseModel(self, datasheet): 'ts_cols': time_cols, 'ts_freq': tsFrequency(ts_kind, leaf), 'ts_format': datetimeFormat(), - } + } if isinstance(leaf, tables.Table): leaf_kind = 'table' else: @@ -246,7 +245,6 @@ def customiseModel(self, datasheet): model.tsFormatter = ts_model.tsFormatter model.data = ts_model.data - def helpAbout(self, parent): """Full description of the plugin. @@ -260,17 +258,17 @@ def helpAbout(self, parent): # Plugin full description desc = {'version': __version__, - 'module_name': os.path.join(os.path.basename(__file__)), - 'folder': os.path.join(os.path.dirname(__file__)), - 'author': 'Vicent Mas ', - 'about_text': translate('TimeFormatterPage', - """ + 'module_name': os.path.join(os.path.basename(__file__)), + 'folder': os.path.join(os.path.dirname(__file__)), + 'author': 'Vicent Mas ', + 'about_text': translate('TimeFormatterPage', + """

Plugin that provides nice string formatting for time fields.

It supports not only native PyTables time datatypes but also time series generated (and stored in PyTables tables) via Pandas and scikits.timeseries packages. """, - 'Text of an About plugin message box')} + 'Text of an About plugin message box')} self.about_page = AboutPage(desc, parent) # We need to install the event filter because the Preferences dialog @@ -313,7 +311,6 @@ def __init__(self, model_info, ts_info, parent=None): else: self.data = self.array_data - def table_data(self, index, role=QtCore.Qt.DisplayRole): """Returns the data stored under the given role for the item referred to by the index. @@ -340,7 +337,6 @@ def table_data(self, index, role=QtCore.Qt.DisplayRole): return None - def array_data(self, index, role=QtCore.Qt.DisplayRole): """Returns the data stored under the given role for the item referred to by the index. @@ -361,11 +357,10 @@ def array_data(self, index, role=QtCore.Qt.DisplayRole): return self.tsFormatter(cell) if role == QtCore.Qt.TextAlignmentRole: - return QtCore.Qt.AlignLeft|QtCore.Qt.AlignTop + return QtCore.Qt.AlignLeft | QtCore.Qt.AlignTop return None - def timeFormatter(self): """Return the function to be used for formatting time series. """ @@ -380,7 +375,6 @@ def timeFormatter(self): time_formatter = self.formatPyTablesTS return time_formatter - def formatPyTablesTS(self, content): """ Format a given date in a user friendly way. @@ -396,7 +390,6 @@ def formatPyTablesTS(self, content): except ValueError: return content - def formatPandasTS(self, content): """Format a given date in a user friendly way. @@ -405,14 +398,13 @@ def formatPandasTS(self, content): :Parameter content: the content of the table cell being formatted """ - + # ImportError if pandas not installed! date = pd.Timestamp(int(content)) try: return date.strftime(self.ts_format) except ValueError: return content - def formatScikitsTS(self, content): """Format a given date in a user friendly way. diff --git a/vitables/vttables/leaf_view.py b/vitables/vttables/leaf_view.py index b842feb6..990806c7 100644 --- a/vitables/vttables/leaf_view.py +++ b/vitables/vttables/leaf_view.py @@ -288,8 +288,8 @@ def subSingleStep(self): # last row of the viewport. model.loadData( buffer_start + page_step - table_rows, table_rows) - self.scrollTo( - model.index(buffer_start + page_step - model.start - 1, 0), + self.scrollTo(model.index( + buffer_start + page_step - model.start - 1, 0), QtWidgets.QAbstractItemView.PositionAtBottom) self.updateView() else: From d5dc8a6cf2ee28a67d1d9a04053f4df50438b782 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Wed, 14 Jun 2017 02:26:58 +0200 Subject: [PATCH 10/13] style(tabmodel): reodred methods --- vitables/vttables/leaf_model.py | 94 +++++++++++++++++---------------- 1 file changed, 48 insertions(+), 46 deletions(-) diff --git a/vitables/vttables/leaf_model.py b/vitables/vttables/leaf_model.py index b0cd5319..35567837 100644 --- a/vitables/vttables/leaf_model.py +++ b/vitables/vttables/leaf_model.py @@ -120,6 +120,54 @@ def __init__(self, leaf, parent=None): super(LeafModel, self).__init__(parent) + def columnCount(self, index=QtCore.QModelIndex()): + """The number of columns of the given model index. + + Overridden to return 0 for valid indices because they have no children; + otherwise return the total number of *columns* exposed by the model. + + :param index: + the model index being inspected. + """ + + return 0 if index.isValid() else self.numcols + + def rowCount(self, index=QtCore.QModelIndex()): + """The number of columns for the children of the given index. + + Overridden to return 0 for valid indices because they have no children; + otherwise return the total number of *rows* exposed by the model. + + :Parameter index: the model index being inspected. + """ + + return 0 if index.isValid() else self.numrows + + def loadData(self, start, length): + """Load the model with fresh data from the buffer. + + :param start: + the document row that is the first row of the chunk. + :param length: + the buffer size, i.e. the number of rows to be read. + + :return: + a tuple with tested values for the parameters of the read method + """ + + # Enforce scrolling limits. + # + start = max(start, 0) + stop = min(start + length, self.leaf_numrows) + + # Ensure buffer filled when scrolled beyond bottom. + # + actual_start = stop - self.numrows + start = max(min(actual_start, start), 0) + + self.rbuffer.readBuffer(start, stop) + self.start = start + def headerData(self, section, orientation, role): """Returns the data for the given role and section in the header with the specified orientation. @@ -176,49 +224,3 @@ def data(self, index, role=QtCore.Qt.DisplayRole): return QtCore.Qt.AlignLeft | QtCore.Qt.AlignTop return None - - def columnCount(self, index=QtCore.QModelIndex()): - """The number of columns of the given model index. - - Overridden to return 0 for valid indices because they have no children; - otherwise return the total number of *columns* exposed by the model. - - :param index: - the model index being inspected. - """ - - return 0 if index.isValid() else self.numcols - - def rowCount(self, index=QtCore.QModelIndex()): - """The number of columns for the children of the given index. - - Overridden to return 0 for valid indices because they have no children; - otherwise return the total number of *rows* exposed by the model. - - :Parameter index: the model index being inspected. - """ - - return 0 if index.isValid() else self.numrows - - def loadData(self, start, length): - """Load the model with fresh chunk from the underlying leaf. - - :param start: - The first row (within the total nrows) of the chunk to read. - :param length: - The buffer size, i.e. the number of rows to be read. - """ - - # Enforce scrolling limits. - # - start = max(start, 0) - stop = min(start + length, self.leaf_numrows) - assert stop >= start, (self.numrows, start, stop, length) - - # Ensure buffer filled when scrolled beyond bottom. - # - actual_start = stop - self.numrows - start = max(min(actual_start, start), 0) - - self.rbuffer.readBuffer(start, stop) - self.start = start From f2679e3ed23834c17528c812e3842bbd0bfdfa07 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Wed, 14 Jun 2017 15:26:02 +0200 Subject: [PATCH 11/13] style(tabview): shorter names for model-item attributes --- vitables/vttables/leaf_view.py | 47 ++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/vitables/vttables/leaf_view.py b/vitables/vttables/leaf_view.py index 990806c7..f4b70052 100644 --- a/vitables/vttables/leaf_view.py +++ b/vitables/vttables/leaf_view.py @@ -38,8 +38,11 @@ from qtpy import QtGui from qtpy import QtWidgets -import vitables.vttables.scrollbar as scrollbar import vitables.vttables.leaf_delegate as leaf_delegate +import vitables.vttables.scrollbar as scrollbar + + +_aiv = QtWidgets.QAbstractItemView class LeafView(QtWidgets.QTableView): @@ -63,11 +66,11 @@ def __init__(self, tmodel, parent=None): self.tmodel = tmodel # This is a MUST self.leaf_numrows = leaf_numrows = self.tmodel.leaf_numrows self.selection_model = self.selectionModel() - self.setSelectionMode(QtWidgets.QAbstractItemView.SingleSelection) - self.setSelectionBehavior(QtWidgets.QAbstractItemView.SelectItems) + self.setSelectionMode(_aiv.SingleSelection) + self.setSelectionBehavior(_aiv.SelectItems) # Setup the actual vertical scrollbar - self.setVerticalScrollMode(QtWidgets.QAbstractItemView.ScrollPerItem) + self.setVerticalScrollMode(_aiv.ScrollPerItem) self.vscrollbar = self.verticalScrollBar() self.setModel(tmodel) @@ -196,7 +199,7 @@ def navigateWithMouse(self, slider_action): 4: self.subPageStep, 7: self.dragSlider } - if not slider_action in actions.keys(): + if slider_action not in actions.keys(): return # Navigate the data dealing with buffer faults actions[slider_action]() @@ -248,7 +251,7 @@ def addSingleStep(self): self.updateView() self.scrollTo( model.index(new_start - model.start, 0), - QtWidgets.QAbstractItemView.PositionAtTop) + _aiv.PositionAtTop) else: self.vscrollbar.triggerAction(1) @@ -270,7 +273,7 @@ def addPageStep(self): self.updateView() self.scrollTo( model.index(new_start - model.start, 0), - QtWidgets.QAbstractItemView.PositionAtTop) + _aiv.PositionAtTop) else: self.vscrollbar.triggerAction(3) @@ -288,9 +291,9 @@ def subSingleStep(self): # last row of the viewport. model.loadData( buffer_start + page_step - table_rows, table_rows) - self.scrollTo(model.index( - buffer_start + page_step - model.start - 1, 0), - QtWidgets.QAbstractItemView.PositionAtBottom) + self.scrollTo( + model.index(buffer_start + page_step - model.start - 1, 0), + _aiv.PositionAtBottom) self.updateView() else: self.vscrollbar.triggerAction(2) @@ -314,7 +317,7 @@ def subPageStep(self): self.scrollTo( model.index( buffer_start + first_vp_row - model.start, 0), - QtWidgets.QAbstractItemView.PositionAtBottom) + _aiv.PositionAtBottom) else: self.vscrollbar.triggerAction(4) @@ -365,7 +368,7 @@ def dragSlider(self): else: self.scrollTo( model.index(row - model.start, 0), - QtWidgets.QAbstractItemView.PositionAtTop) + _aiv.PositionAtTop) def topBF(self, value, row): """Going out of buffer when browsing upwards. @@ -382,13 +385,13 @@ def topBF(self, value, row): if value == self.tricky_vscrollbar.minimum(): start = 0 position = 0 - hint = QtWidgets.QAbstractItemView.PositionAtTop + hint = _aiv.PositionAtTop self.vscrollbar.triggerAction( QtWidgets.QAbstractSlider.SliderToMinimum) else: start = row - table_rows position = table_rows - 1 - hint = QtWidgets.QAbstractItemView.PositionAtBottom + hint = _aiv.PositionAtBottom self.tmodel.loadData(start, table_rows) self.updateView() @@ -410,13 +413,13 @@ def bottomBF(self, value, row): row = self.leaf_numrows - 1 start = self.leaf_numrows - table_rows position = table_rows - 1 - hint = QtWidgets.QAbstractItemView.PositionAtBottom + hint = _aiv.PositionAtBottom self.vscrollbar.triggerAction( QtWidgets.QAbstractSlider.SliderToMinimum) else: start = row position = 0 - hint = QtWidgets.QAbstractItemView.PositionAtTop + hint = _aiv.PositionAtTop self.tmodel.loadData(start, table_rows) self.updateView() @@ -465,7 +468,7 @@ def wheelDown(self, event): model.loadData(new_start, table_rows) self.updateView() self.scrollTo(model.index(new_start - model.start, 0), - QtWidgets.QAbstractItemView.PositionAtTop) + _aiv.PositionAtTop) else: QtCore.QCoreApplication.sendEvent(self.vscrollbar, event) @@ -489,7 +492,7 @@ def wheelUp(self, event): self.scrollTo( model.index( new_start + table_rows - model.start - 1, 0), - QtWidgets.QAbstractItemView.PositionAtBottom) + _aiv.PositionAtBottom) else: QtCore.QCoreApplication.sendEvent(self.vscrollbar, event) @@ -616,7 +619,7 @@ def upKeyPressEvent(self, event): index = model.index(row, buffer_column) self.setCurrentIndex(index) self.scrollTo(index, - QtWidgets.QAbstractItemView.PositionAtTop) + _aiv.PositionAtTop) else: QtWidgets.QTableView.keyPressEvent(self, event) @@ -647,7 +650,7 @@ def pageUpKeyPressEvent(self, event): index = model.index(row, buffer_column) self.setCurrentIndex(index) self.scrollTo(index, - QtWidgets.QAbstractItemView.PositionAtTop) + _aiv.PositionAtTop) else: QtWidgets.QTableView.keyPressEvent(self, event) @@ -679,7 +682,7 @@ def downKeyPressEvent(self, event): index = model.index(row, buffer_column) self.setCurrentIndex(index) self.scrollTo(index, - QtWidgets.QAbstractItemView.PositionAtBottom) + _aiv.PositionAtBottom) else: QtWidgets.QTableView.keyPressEvent(self, event) @@ -711,7 +714,7 @@ def pageDownKeyPressEvent(self, event): index = model.index(row, buffer_column) self.setCurrentIndex(index) self.scrollTo(index, - QtWidgets.QAbstractItemView.PositionAtBottom) + _aiv.PositionAtBottom) else: QtWidgets.QTableView.keyPressEvent(self, event) From 4ad73cc96ce5dd3a042a485e9d0cd3ff6df2ef5b Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Wed, 14 Jun 2017 02:51:49 +0200 Subject: [PATCH 12/13] refact(table): encapsulate buf.getCell() & improve logging + Centralize error-reporting for cell indexing-errors. + Logging: + Restore start offset-logging on cell indexing errors. + Class-logger-->module-var, not to waste loggers with each new object. + Stop specifying log-levels (must be done to all loggers). --- vitables/plugins/timeseries/time_series.py | 14 +++++---- vitables/vttables/buffer.py | 36 ++++++---------------- vitables/vttables/datasheet.py | 12 +++----- vitables/vttables/leaf_model.py | 23 +++++++++++--- 4 files changed, 40 insertions(+), 45 deletions(-) diff --git a/vitables/plugins/timeseries/time_series.py b/vitables/plugins/timeseries/time_series.py index 6975e0e1..77fda106 100644 --- a/vitables/plugins/timeseries/time_series.py +++ b/vitables/plugins/timeseries/time_series.py @@ -231,7 +231,7 @@ def customiseModel(self, datasheet): leaf_kind = 'array' model_info = { 'leaf_kind': leaf_kind, - 'rbuffer': model.rbuffer, + 'model': model, 'numrows': model.rowCount(), 'formatContent': model.formatContent, } @@ -298,7 +298,7 @@ def __init__(self, model_info, ts_info, parent=None): self.ts_freq = ts_info['ts_freq'] self.ts_format = ts_info['ts_format'] # Attributes required by the data() method - self.rbuffer = model_info['rbuffer'] + self.model = model_info['model'] self.numrows = model_info['numrows'] self.ts_cols = ts_info['ts_cols'] self.formatContent = model_info['formatContent'] @@ -322,12 +322,13 @@ def table_data(self, index, role=QtCore.Qt.DisplayRole): - `index`: the index of a data item - `role`: the role being returned """ + row, col = index.row(), index.column() - if not index.isValid() or not (0 <= index.row() < self.nrows): + if not index.isValid() or not (0 <= row < self.nrows): return None if role == QtCore.Qt.DisplayRole: - cell = self.rbuffer.getCell(index.row(), index.column()) + cell = self.model.cell(row, col) if index.column() in self.ts_cols: return self.tsFormatter(cell) return self.formatContent(cell) @@ -348,12 +349,13 @@ def array_data(self, index, role=QtCore.Qt.DisplayRole): - `index`: the index of a data item - `role`: the role being returned """ + row, col = index.row(), index.column() - if not index.isValid() or not (0 <= index.row() < self.nrows): + if not index.isValid() or not (0 <= row < self.nrows): return None if role == QtCore.Qt.DisplayRole: - cell = self.rbuffer.getCell(index.row(), index.column()) + cell = self.model.cell(row, col) return self.tsFormatter(cell) if role == QtCore.Qt.TextAlignmentRole: diff --git a/vitables/vttables/buffer.py b/vitables/vttables/buffer.py index e9ae953c..5571ee44 100644 --- a/vitables/vttables/buffer.py +++ b/vitables/vttables/buffer.py @@ -48,6 +48,8 @@ warnings.filterwarnings('ignore', category=tables.FlavorWarning) warnings.filterwarnings('ignore', category=tables.NaturalNameWarning) +log = logging.getLogger(__name__) + class Buffer(object): """Buffer used to access the real data contained in `PyTables` datasets. @@ -81,9 +83,6 @@ def __init__(self, leaf): Initializes the buffer. """ - self.logger = logging.getLogger(__name__) - self.logger.setLevel(logging.INFO) - self.leaf = leaf # The numpy array where read data will be stored @@ -161,7 +160,7 @@ def isDataSourceReadable(self): except tables.HDF5ExtError: readable = False # TODO: Fix error msg to include exception or merge with below. - self.logger.error( + log.error( translate('Buffer', """Problems reading records. The dataset """ """seems to be compressed with the {0} library. """ @@ -171,7 +170,7 @@ def isDataSourceReadable(self): self.leaf.filters.complib)) except ValueError as e: readable = False - self.logger.error( + log.error( translate('Buffer', 'Data read error: {}', 'A dataset read error').format(e.message)) return readable @@ -204,7 +203,7 @@ def readBuffer(self, start, stop): data = self.leaf.read(start, stop) except tables.HDF5ExtError: # TODO: Fix error msg to include exception. - self.logger.error( + log.error( translate('Buffer', """\nError: problems reading records. """ """The dataset maybe corrupted.""", 'A dataset readability error')) @@ -228,12 +227,7 @@ def scalarCell(self, row, col): :Returns: the cell at position `(row, col)` of the document """ - - try: - return self.chunk[()] - except IndexError: - self.logger.error('IndexError! row, column: {1}, {2}' - .format(row, col)) + return self.chunk[()] def vectorCell(self, row, col): """ @@ -260,11 +254,7 @@ def vectorCell(self, row, col): # get the right chunk element. # chunk = [row0, row1, row2, ..., rowN] # and columns can be read from a given row using indexing notation - try: - return self.chunk[row] - except IndexError: - self.logger.error('IndexError! row, column: {1}, {2}' - .format(row, col)) + return self.chunk[row] def EArrayCell(self, row, col): """ @@ -287,11 +277,7 @@ def EArrayCell(self, row, col): # and columns can be read from a given row using indexing notation # TODO: this method should be improved as it requires to read the # whola array keeping the read data in memory - try: - return self.leaf.read()[row] - except IndexError: - self.logger.error('IndexError! row, column: {1}, {2}' - .format(row, col)) + return self.leaf.read()[row] def arrayCell(self, row, col): """ @@ -316,8 +302,4 @@ def arrayCell(self, row, col): # For tables we have # chunk = [nestedrecord0, nestedrecord1, ..., nestedrecordN] # and fields can be read from nestedrecordJ using indexing notation - try: - return self.chunk[row][col] - except IndexError: - self.logger.error('IndexError! row, column: {1}, {2}' - .format(row, col)) + return self.chunk[row][col] diff --git a/vitables/vttables/datasheet.py b/vitables/vttables/datasheet.py index eb94e40c..5720d41b 100644 --- a/vitables/vttables/datasheet.py +++ b/vitables/vttables/datasheet.py @@ -94,7 +94,6 @@ def __init__(self, index): self.aboutToActivate.connect(self.syncTreeView) self.leaf_view.doubleClicked.connect(self.zoomCell) - def closeEvent(self, event): """Close the window cleanly with the close button of the title bar. @@ -112,7 +111,6 @@ def closeEvent(self, event): if self.vtgui.workspace.subWindowList() == []: self.vtgui.dbs_tree_view.setFocus(True) - def focusInEvent(self, event): """Specialised handler for focus events. @@ -128,7 +126,6 @@ def focusInEvent(self, event): self.syncTreeView() self.setFocus(True) - def syncTreeView(self): """ If the view is activated select its leaf in the tree of databases view. @@ -143,7 +140,6 @@ def syncTreeView(self): self.vtgui.dbs_tree_view.setCurrentIndex( QtCore.QModelIndex(self.pindex)) - def zoomCell(self, index): """Display the inner dimensions of a cell. @@ -153,7 +149,7 @@ def zoomCell(self, index): row = index.row() column = index.column() tmodel = index.model() - data = tmodel.rbuffer.getCell(row, column) + data = tmodel.cell(row, column) # The title of the zoomed view node = self.dbt_leaf @@ -161,10 +157,10 @@ def zoomCell(self, index): if node.node_kind == 'table': col = info.columns_names[column] title = '{0}: {1}[{2}]'.format(node.name, col, - tmodel.start + row) + tmodel.start + row) else: title = '{0}: ({1},{2})'.format(node.name, - tmodel.start + row, column) + tmodel.start + row, column) zoom_cell.ZoomCell(data, title, self.vtgui.workspace, - self.dbt_leaf) + self.dbt_leaf) diff --git a/vitables/vttables/leaf_model.py b/vitables/vttables/leaf_model.py index 35567837..e94ea13f 100644 --- a/vitables/vttables/leaf_model.py +++ b/vitables/vttables/leaf_model.py @@ -26,17 +26,19 @@ __docformat__ = 'restructuredtext' +import logging import vitables.utils +from vitables.vttables import buffer from qtpy import QtCore import tables -from vitables.vttables import buffer - #: The maximum number of rows to be read from the data source. CHUNK_SIZE = 10000 +log = logging.getLogger(__name__) + class LeafModel(QtCore.QAbstractTableModel): """ @@ -212,15 +214,28 @@ def data(self, index, role=QtCore.Qt.DisplayRole): - `index`: the index of a data item - `role`: the role being returned """ + row, col = index.row(), index.column() - if not index.isValid() or not (0 <= index.row() < self.numrows): + if not index.isValid() or not (0 <= row < self.numrows): return None if role == QtCore.Qt.DisplayRole: - cell = self.rbuffer.getCell(index.row(), index.column()) + cell = self.cell(row, col) return self.formatContent(cell) if role == QtCore.Qt.TextAlignmentRole: return QtCore.Qt.AlignLeft | QtCore.Qt.AlignTop return None + + def cell(self, row, col): + """ + Returns the contents of a cell. + + :return: none to disable zooming. + """ + try: + return self.rbuffer.getCell(row, col) + except IndexError: + log.error('IndexError! buffer start: {0} row, column: ' + '{1}, {2}'.format(self.start, row, col)) From ae05d19d3658783c587aad41edd3c9fbe2cf66cd Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Wed, 14 Jun 2017 15:23:37 +0200 Subject: [PATCH 13/13] feat(datasheet): don't zoom-cells if empty --- vitables/vttables/datasheet.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vitables/vttables/datasheet.py b/vitables/vttables/datasheet.py index 5720d41b..317f5005 100644 --- a/vitables/vttables/datasheet.py +++ b/vitables/vttables/datasheet.py @@ -150,6 +150,8 @@ def zoomCell(self, index): column = index.column() tmodel = index.model() data = tmodel.cell(row, column) + if data is None: + return # The title of the zoomed view node = self.dbt_leaf