[mis_builder] generalize auto expands

mis_builder/models/aep.py

@@ -14,6 +14,8 @@
 
 _DOMAIN_START_RE = re.compile(r"\(|(['\"])[!&|]\1")
 
+UNCLASSIFIED_ROW_DETAIL = "other"
+
 
 def _is_domain(s):
     """Test if a string looks like an Odoo domain"""
@@ -300,16 +302,14 @@ def do_queries(
         date_to,
         additional_move_line_filter=None,
         aml_model=None,
+        auto_expand_col_name=None,
     ):
         """Query sums of debit and credit for all accounts and domains
         used in expressions.
 
         This method must be executed after done_parsing().
         """
-        if not aml_model:
-            aml_model = self.env["account.move.line"]
-        else:
-            aml_model = self.env[aml_model]
+        aml_model = self.env[aml_model or "account.move.line"]
         aml_model = aml_model.with_context(active_test=False)
         company_rates = self._get_company_rates(date_to)
         # {(domain, mode): {account_id: (debit, credit)}}
@@ -330,13 +330,16 @@ def do_queries(
             domain.append(("account_id", "in", self._map_account_ids[key]))
             if additional_move_line_filter:
                 domain.extend(additional_move_line_filter)
+
+            get_fields = ["debit", "credit", "account_id", "company_id"]
+            group_by_fields = ["account_id", "company_id"]
+            if auto_expand_col_name:
+                get_fields = [auto_expand_col_name] + get_fields
+                group_by_fields = [auto_expand_col_name] + group_by_fields
+
             # fetch sum of debit/credit, grouped by account_id
-            accs = aml_model.read_group(
-                domain,
-                ["debit", "credit", "account_id", "company_id"],
-                ["account_id", "company_id"],
-                lazy=False,
-            )
+            accs = aml_model.read_group(domain, get_fields, group_by_fields, lazy=False)
+
             for acc in accs:
                 rate, dp = company_rates[acc["company_id"][0]]
                 debit = acc["debit"] or 0.0
@@ -346,19 +349,45 @@ def do_queries(
                 ):
                     # in initial mode, ignore accounts with 0 balance
                     continue
-                self._data[key][acc["account_id"][0]] = (debit * rate, credit * rate)
+                if (
+                    auto_expand_col_name
+                    and auto_expand_col_name in acc
+                    and acc[auto_expand_col_name]
+                ):
+                    rdi_id = acc[auto_expand_col_name][0]
+                else:
+                    rdi_id = UNCLASSIFIED_ROW_DETAIL
+                if not self._data[key].get(rdi_id, False):
+                    self._data[key][rdi_id] = defaultdict(dict)
+                self._data[key][rdi_id][acc["account_id"][0]] = (
+                    debit * rate,
+                    credit * rate,
+                )
         # compute ending balances by summing initial and variation
         for key in ends:
             domain, mode = key
             initial_data = self._data[(domain, self.MODE_INITIAL)]
             variation_data = self._data[(domain, self.MODE_VARIATION)]
-            account_ids = set(initial_data.keys()) | set(variation_data.keys())
-            for account_id in account_ids:
-                di, ci = initial_data.get(account_id, (AccountingNone, AccountingNone))
-                dv, cv = variation_data.get(
-                    account_id, (AccountingNone, AccountingNone)
+            rdis = set(initial_data.keys()) | set(variation_data.keys())
+            for rdi in rdis:
+                if not initial_data.get(rdi, False):
+                    initial_data[rdi] = defaultdict(dict)
+                if not variation_data.get(rdi, False):
+                    variation_data[rdi] = defaultdict(dict)
+                if not self._data[key].get(rdi, False):
+                    self._data[key][rdi] = defaultdict(dict)
+
+                account_ids = set(initial_data[rdi].keys()) | set(
+                    variation_data[rdi].keys()
                 )
-                self._data[key][account_id] = (di + dv, ci + cv)
+                for account_id in account_ids:
+                    di, ci = initial_data[rdi].get(
+                        account_id, (AccountingNone, AccountingNone)
+                    )
+                    dv, cv = variation_data[rdi].get(
+                        account_id, (AccountingNone, AccountingNone)
+                    )
+                    self._data[key][rdi][account_id] = (di + dv, ci + cv)
 
     def replace_expr(self, expr):
         """Replace accounting variables in an expression by their amount.
@@ -371,23 +400,25 @@ def replace_expr(self, expr):
         def f(mo):
             field, mode, acc_domain, ml_domain = self._parse_match_object(mo)
             key = (ml_domain, mode)
-            account_ids_data = self._data[key]
+            rdi_ids_data = self._data[key]
             v = AccountingNone
             account_ids = self._account_ids_by_acc_domain[acc_domain]
-            for account_id in account_ids:
-                debit, credit = account_ids_data.get(
-                    account_id, (AccountingNone, AccountingNone)
-                )
-                if field == "bal":
-                    v += debit - credit
-                elif field == "pbal" and debit >= credit:
-                    v += debit - credit
-                elif field == "nbal" and debit < credit:
-                    v += debit - credit
-                elif field == "deb":
-                    v += debit
-                elif field == "crd":
-                    v += credit
+            for rdi in rdi_ids_data:
+                account_ids_data = self._data[key][rdi]
+                for account_id in account_ids:
+                    debit, credit = account_ids_data.get(
+                        account_id, (AccountingNone, AccountingNone)
+                    )
+                    if field == "bal":
+                        v += debit - credit
+                    elif field == "pbal" and debit >= credit:
+                        v += debit - credit
+                    elif field == "nbal" and debit < credit:
+                        v += debit - credit
+                    elif field == "deb":
+                        v += debit
+                    elif field == "crd":
+                        v += credit
             # in initial balance mode, assume 0 is None
             # as it does not make sense to distinguish 0 from "no data"
             if (
@@ -401,11 +432,11 @@ def f(mo):
         return self._ACC_RE.sub(f, expr)
 
     def replace_exprs_by_account_id(self, exprs):
-        """Replace accounting variables in a list of expression
-        by their amount, iterating by accounts involved in the expression.
+        """This method is depreciated and replaced by replace_exprs_by_row_detail.
 
+        Replace accounting variables in a list of expression
+        by their amount, iterating by accounts involved in the expression.
         yields account_id, replaced_expr
-
         This method must be executed after do_queries().
         """
 
@@ -417,7 +448,7 @@ def f(mo):
             if account_id not in self._account_ids_by_acc_domain[acc_domain]:
                 return "(AccountingNone)"
             # here we know account_id is involved in acc_domain
-            account_ids_data = self._data[key]
+            account_ids_data = self._data[key][UNCLASSIFIED_ROW_DETAIL]
             debit, credit = account_ids_data.get(
                 account_id, (AccountingNone, AccountingNone)
             )
@@ -452,14 +483,66 @@ def f(mo):
             for mo in self._ACC_RE.finditer(expr):
                 field, mode, acc_domain, ml_domain = self._parse_match_object(mo)
                 key = (ml_domain, mode)
-                account_ids_data = self._data[key]
+                account_ids_data = self._data[key][UNCLASSIFIED_ROW_DETAIL]
                 for account_id in self._account_ids_by_acc_domain[acc_domain]:
                     if account_id in account_ids_data:
                         account_ids.add(account_id)
 
         for account_id in account_ids:
             yield account_id, [self._ACC_RE.sub(f, expr) for expr in exprs]
 
+    def replace_exprs_by_row_detail(self, exprs):
+        """Replace accounting variables in a list of expression
+        by their amount, iterating by accounts involved in the expression.
+
+        yields account_id, replaced_expr
+
+        This method must be executed after do_queries().
+        """
+
+        def f(mo):
+            field, mode, acc_domain, ml_domain = self._parse_match_object(mo)
+            key = (ml_domain, mode)
+            v = AccountingNone
+            account_ids_data = self._data[key][rdi_id]
+            account_ids = self._account_ids_by_acc_domain[acc_domain]
+
+            for account_id in account_ids:
+                debit, credit = account_ids_data.get(
+                    account_id, (AccountingNone, AccountingNone)
+                )
+                if field == "bal":
+                    v += debit - credit
+                elif field == "pbal" and debit >= credit:
+                    v += debit - credit
+                elif field == "nbal" and debit < credit:
+                    v += debit - credit
+                elif field == "deb":
+                    v += debit
+                elif field == "crd":
+                    v += credit
+            # in initial balance mode, assume 0 is None
+            # as it does not make sense to distinguish 0 from "no data"
+            if (
+                v is not AccountingNone
+                and mode in (self.MODE_INITIAL, self.MODE_UNALLOCATED)
+                and float_is_zero(v, precision_digits=self.dp)
+            ):
+                v = AccountingNone
+            return "(" + repr(v) + ")"
+
+        rdi_ids = set()
+        for expr in exprs:
+            for mo in self._ACC_RE.finditer(expr):
+                field, mode, acc_domain, ml_domain = self._parse_match_object(mo)
+                key = (ml_domain, mode)
+                rdis_data = self._data[key]
+                for rdi_id in rdis_data.keys():
+                    rdi_ids.add(rdi_id)
+
+        for rdi_id in rdi_ids:
+            yield rdi_id, [self._ACC_RE.sub(f, expr) for expr in exprs]
+
     @classmethod
     def _get_balances(cls, mode, companies, date_from, date_to):
         expr = "deb{mode}[], crd{mode}[]".format(mode=mode)
@@ -470,7 +553,10 @@ def _get_balances(cls, mode, companies, date_from, date_to):
         aep.parse_expr(expr)
         aep.done_parsing()
         aep.do_queries(date_from, date_to)
-        return aep._data[((), mode)]
+
+        return aep._data[((), mode)].get(UNCLASSIFIED_ROW_DETAIL, {})
+        # to keep compatibility, we give the UNCLASSIFIED_ROW_DETAIL
+        # (expecting that auto_expand_col_names=None was given to do_queries )
 
     @classmethod
     def get_balances_initial(cls, companies, date):

mis_builder/models/expression_evaluator.py

@@ -20,13 +20,14 @@ def __init__(
         self.aml_model = aml_model
         self._aep_queries_done = False
 
-    def aep_do_queries(self):
+    def aep_do_queries(self, auto_expand_col_name=None):
         if self.aep and not self._aep_queries_done:
             self.aep.do_queries(
                 self.date_from,
                 self.date_to,
                 self.additional_move_line_filter,
                 self.aml_model,
+                auto_expand_col_name,
             )
             self._aep_queries_done = True
 
@@ -50,6 +51,7 @@ def eval_expressions(self, expressions, locals_dict):
                 drilldown_args.append(None)
         return vals, drilldown_args, name_error
 
+    # we keep it for backward compatibility
     def eval_expressions_by_account(self, expressions, locals_dict):
         if not self.aep:
             return
@@ -66,3 +68,20 @@ def eval_expressions_by_account(self, expressions, locals_dict):
                 else:
                     drilldown_args.append(None)
             yield account_id, vals, drilldown_args, name_error
+
+    def eval_expressions_by_row_detail(self, expressions, locals_dict):
+        if not self.aep:
+            return
+        exprs = [e and e.name or "AccountingNone" for e in expressions]
+        for rdi_id, replaced_exprs in self.aep.replace_exprs_by_row_detail(exprs):
+            vals = []
+            drilldown_args = []
+            name_error = False
+            for expr, replaced_expr in zip(exprs, replaced_exprs):
+                val = mis_safe_eval(replaced_expr, locals_dict)
+                vals.append(val)
+                if replaced_expr != expr:
+                    drilldown_args.append({"expr": expr, "row_detail": rdi_id})
+                else:
+                    drilldown_args.append(None)
+            yield rdi_id, vals, drilldown_args, name_error