Dfview

exetera/core/abstract_types.py

@@ -496,3 +496,18 @@ def ordered_merge_right(self, right_on, left_on,
                             left_field_sources=tuple(), right_field_sinks=None,
                             right_to_left_map=None, right_unique=False, left_unique=False):
         raise NotImplementedError()
+
+
+class SubjectObserver(ABC):
+    def attach(self, observer):
+        raise NotImplementedError()
+
+    def detach(self, observer):
+        raise NotImplementedError()
+
+    def notify(self, msg=None):
+        raise NotImplementedError()
+
+    def update(self, subject, msg=None):
+        raise NotImplementedError()
+

exetera/core/dataframe.py

@@ -107,27 +107,72 @@ def add(self,
             nfield.data.write(field.data[:])
         self._columns[dname] = nfield
 
-    def add_view(self, field: fld.Field):
+    def _add_view(self, field: fld.Field, filter: np.ndarray = None):
         """
+        Internal function called by apply_filter to add a field view into the dataframe.
+
+        :param field: The field to apply filter to.
+        :param filter: The filter to apply.
+        :return: The field view.
 
         """
+        # add view
         if isinstance(field, fld.NumericField):
             view = fld.NumericField(field._session, field._field, self, write_enabled=True)
-            view.data = field.data
-
+        elif isinstance(field, fld.CategoricalField):
+            view = fld.CategoricalField(field._session, field._field, self, write_enabled=True)
+        elif isinstance(field, fld.TimestampField):
+            view = fld.TimestampField(field._session, field._field, self, write_enabled=True)
+        elif isinstance(field, fld.FixedStringField):
+            view = fld.FixedStringField(field._session, field._field, self, write_enabled=True)
+        elif isinstance(field, fld.IndexedStringField):
+            view = fld.IndexedStringField(field._session, field._field, self, write_enabled=True)
+
+        field.attach(view)
         self._columns[view.name] = view
+
+        # add filter
+        if filter is not None:
+            nformat = 'int32' if filter[-1] < 2 ** 31 - 1 else 'int64'
+            filter_name = view.name
+            if filter_name not in self._filters_grp.keys():
+                fld.numeric_field_constructor(self._dataset.session, self._filters_grp, filter_name, nformat)
+                filter_field = fld.NumericField(self._dataset.session, self._filters_grp[filter_name], self,
+                                                write_enabled=True)
+                filter_field.data.write(filter)
+            else:
+                filter_field = fld.NumericField(self._dataset.session, self._filters_grp[filter_name], self,
+                                                write_enabled=True)
+                if nformat not in filter_field._fieldtype:
+                    filter_field = filter_field.astype(nformat)
+                filter_field.data.clear()
+                filter_field.data.write(filter)
+
+            view._filter_wrapper = filter_field.data
+
         return self._columns[view.name]
 
-    # def add_reference(self, field: fld.Field):
+    # def change_filter(self, field: fld.Field, filter: np.ndarray):
+    #     """
+    #
+    #     :param field:
+    #     :param filter:
+    #     :return:
     #     """
-    #     Add a field without coping the data over the HDF5 group.
-    #     :param field: field to be constructed in this dataframe.
+    #     pass
+    #
+    # def remove_filter(self, field: Union[str, fld.Field]):
+    #     """
+    #     Remove filter from this dataframe specified by the field or field name.
     #     """
-    #     if isinstance(field, fld.NumericField):
-    #         fld.numeric_field_constructor(self._dataset.session, self, field.name, field._nformat)
-    #         nfield = fld.NumericField(self._dataset.session, field._field, self, write_enabled=True)
-    #         self._columns[field.name] = nfield
-    #         return self._columns[field.name]
+    #     if not isinstance(field, str) and not isinstance(field, fld.Field):
+    #         raise TypeError("The target field should be type field or string (name of the field in this dataframe).")
+    #
+    #     name = field if isinstance(field, str) else field.name
+    #     if name not in self._columns:
+    #         raise ValueError("The target field is not in this dataframe.")
+    #     else:
+    #         del self._filters_grp[name]
 
     def drop(self,
              name: str):
@@ -136,10 +181,9 @@ def drop(self,
 
         :param name: name of field to be dropped
         """
-        if name in self._h5group.keys():
+        del self._columns[name]  # should always be
+        if name in self._h5group.keys():  # in case of reference only
             del self._h5group[name]
-        if name in self._columns.keys():
-            del self._columns[name]
 
     def create_group(self,
                      name: str):
@@ -294,22 +338,6 @@ def contains_field(self, field):
                     return True
             return False
 
-    def _write_filter(self, filter):
-        """
-
-        """
-        nformat = 'int32' if filter[-1] < 2 ** 31 - 1 else 'int64'
-        filter_name = '_filter'
-        if filter_name not in self._filters_grp.keys():
-            fld.numeric_field_constructor(self._dataset.session, self._filters_grp, filter_name, nformat)
-            filter_field = fld.NumericField(self._dataset.session, self._filters_grp[filter_name], self, write_enabled=True)
-            filter_field.data.write(filter)
-        else:
-            filter_field = fld.NumericField(self._dataset.session, self._filters_grp[filter_name], self, write_enabled=True)
-            if nformat not in filter_field._fieldtype:
-                filter_field = filter_field.astype(nformat)
-            filter_field.data.clear()
-            filter_field.data.write(filter)
 
     def _get_filter_grp(self, field: Union[str, fld.Field]=None):
         """
@@ -318,68 +346,6 @@ def _get_filter_grp(self, field: Union[str, fld.Field]=None):
         filter_name = '_filter'
         return self._filters_grp[filter_name]
 
-    # def set_filter(self, field: Union[str, fld.Field], filter):
-    #     """
-    #     Add or modify a filter of the field.
-    #
-    #     :param field: The target field.
-    #     :param filter: The filter, as list or np.ndarray of indices.
-    #     """
-    #     if not isinstance(field, str) and not isinstance(field, fld.Field):
-    #         raise TypeError("The target field should be type field or string (name of the field in this dataframe).")
-    #
-    #     name = field if isinstance(field, str) else field.name
-    #     if name not in self._columns:
-    #         raise ValueError("The target field is not in this dataframe.")
-    #
-    #     nformat = 'int32' if filter[-1] < 2 ** 31 - 1 else 'int64'
-    #     if name in self._filters_grp.keys():
-    #         filter_field = fld.NumericField(self._dataset.session, self._filters_grp[name], self,
-    #                                         write_enabled=True)
-    #         if nformat not in filter_field._fieldtype:
-    #             filter_field = filter_field.astype(nformat)
-    #         filter_field.data.clear()
-    #         filter_field.data.write(filter)
-    #     else:
-    #         fld.numeric_field_constructor(self._dataset.session, self._filters_grp, name, nformat)
-    #         filter_field = fld.NumericField(self._dataset.session, self._filters_grp[name], self,
-    #                                  write_enabled=True)
-    #         filter_field.data.write(filter)
-    #
-    #     self._columns[name].filter = self._filters_grp[name]
-    #     return filter_field
-
-    def remove_filter(self, field: Union[str, fld.Field]):
-        """
-        Remove filter from this dataframe specified by the field or field name.
-        """
-        if not isinstance(field, str) and not isinstance(field, fld.Field):
-            raise TypeError("The target field should be type field or string (name of the field in this dataframe).")
-
-        name = field if isinstance(field, str) else field.name
-        if name not in self._columns:
-            raise ValueError("The target field is not in this dataframe.")
-        else:
-            del self._filters_grp[name]
-
-    # def get_data(self, field: Union[str, fld.Field]):
-    #     """
-    #     Get the data from a field. The data returned is masked by the filter.
-    #
-    #     """
-    #     if not isinstance(field, str) and not isinstance(field, fld.Field):
-    #         raise TypeError("The target field should be type field or string (name of the field in this dataframe).")
-    #
-    #     name = field if isinstance(field, str) else field.name
-    #     if name not in self.columns.keys():
-    #         raise ValueError("Can not found the field name from this dataframe.")
-    #     else:
-    #         if name in self.filters.keys():
-    #             d_filter = self.filters[name].data[:]
-    #             return self.columns[name].data[d_filter]
-    #         else:
-    #             return self.columns[name].data[:]
-
     def __getitem__(self, name):
         """
         Get a field stored by the field name.
@@ -433,10 +399,10 @@ def __delitem__(self, name):
         if not self.__contains__(name=name):
             raise ValueError("There is no field named '{}' in this dataframe".format(name))
         else:
-            if name in self._h5group.keys():
+            del self._columns[name]  # should always be
+            if name in self._h5group.keys():  # in case of reference only
                 del self._h5group[name]
-            if name in self._columns.keys():
-                del self._columns[name]
+
 
     def delete_field(self, field):
         """
@@ -596,25 +562,18 @@ def apply_filter(self, filter_to_apply, ddf=None):
         :returns: a dataframe contains all the fields filterd, self if ddf is not set
         """
         filter_to_apply_ = val.validate_filter(filter_to_apply)
-        if ddf is not None:
+        if ddf is not None and ddf is not self:
             if not isinstance(ddf, DataFrame):
                 raise TypeError("The destination object must be an instance of DataFrame.")
-            ddf._write_filter(np.where(filter_to_apply_ == True)[0])
+            filter_to_apply_ = filter_to_apply_.nonzero()[0]
             for name, field in self._columns.items():
-                # hard copy
-                # newfld = field.create_like(ddf, name)
-                # field.apply_filter(filter_to_apply_, target=newfld)
-                # soft copy - view
-                newfld = ddf.add_view(field)
-                newfld.filter = ddf._get_filter_grp()
-
+                ddf._add_view(field, filter_to_apply_)
             return ddf
         else:
             for field in self._columns.values():
                 field.apply_filter(filter_to_apply_, in_place=True)
             return self
 
-
     def apply_index(self, index_to_apply, ddf=None):
         """
         Apply an index to all fields in this dataframe, returns \
@@ -638,34 +597,20 @@ def apply_index(self, index_to_apply, ddf=None):
         :param ddf: optional- the destination data frame
         :returns: a dataframe contains all the fields re-indexed, self if ddf is not set
         """
-        if ddf is not None:
+        if ddf is not None and ddf is not self:
             if not isinstance(ddf, DataFrame):
                 raise TypeError("The destination object must be an instance of DataFrame.")
-            if ddf == self:
-                val.validate_all_field_length_in_df(self)
-            for field in self._columns.values():
+            for name, field in self._columns.items():
+                # newfld = field.create_like(ddf, name)
+                # field.apply_index(index_to_apply, target=newfld)
+                ddf._add_view(field, index_to_apply)
+            return ddf
+        else:
+            val.validate_all_field_length_in_df(self)
 
-                if ddf == self:
-                    field.apply_index(index_to_apply, in_place=True)
-                else:
-                    newfld = field.create_like(ddf, field.name)
-                    field.apply_index(index_to_apply, target=newfld)
-        else:  #
-            nformat = 'int32' if index_to_apply[-1] < 2 ** 31 - 1 else 'int64'
             for field in self._columns.values():
-                if field.name in self._filters_grp.keys():
-                    flt_fld = fld.NumericField(self._dataset.session, self._filters_grp[field.name], self,
-                                               write_enabled=True)
-                    if nformat not in flt_fld._fieldtype:
-                        flt_fld = flt_fld.astype(nformat)
-                    flt_fld.data.clear()
-                    flt_fld.data.write(index_to_apply)
-                else:
-                    fld.numeric_field_constructor(self._dataset.session, self._filters_grp, field.name, nformat)
-                    flt_fld = fld.NumericField(self._dataset.session, self._filters_grp[field.name], self,
-                                               write_enabled=True)
-                    flt_fld.data.write(index_to_apply)
-                field.filter = flt_fld._field
+                field.apply_index(index_to_apply, in_place=True)
+            return self
 
 
     def sort_values(self, by: Union[str, List[str]], ddf: DataFrame = None, axis=0, ascending=True, kind='stable'):
@@ -1123,7 +1068,7 @@ def describe(self, include=None, exclude=None, output='terminal'):
     def view(self):
         dfv = self.dataset.create_dataframe(self.name + '_view')
         for f in self.columns.values():
-            dfv.add_view(f)
+            dfv._add_view(f)
         return dfv