Skip to content

Commit

Permalink
Generalized auto detail expansion mechanism
Browse files Browse the repository at this point in the history 
This commit introduce a mechanism to expand automatically a kpi by any field.
  • Loading branch information
@TeoGoddet
TeoGoddet committed Mar 15, 2022
1 parent 034f9d6 commit 3e21d12
Show file tree
Hide file tree
Showing 8 changed files with 333 additions and 104 deletions.
164 changes: 125 additions & 39 deletions mis_builder/models/aep.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

_DOMAIN_START_RE = re.compile(r"\(|(['\"])[!&|]\1")

UNCLASSIFIED_ROW_DETAIL = "other"


def _is_domain(s):
"""Test if a string looks like an Odoo domain"""
Expand Down Expand Up @@ -300,24 +302,22 @@ def do_queries(
date_to,
additional_move_line_filter=None,
aml_model=None,
auto_expand_col_name=None,
):
"""Query sums of debit and credit for all accounts and domains
used in expressions.
This method must be executed after done_parsing().
"""
if not aml_model:
aml_model = self.env["account.move.line"]
else:
aml_model = self.env[aml_model]
aml_model = self.env[aml_model or "account.move.line"]
aml_model = aml_model.with_context(active_test=False)
company_rates = self._get_company_rates(date_to)
# {(domain, mode): {account_id: (debit, credit)}}
self._data = defaultdict(dict)
domain_by_mode = {}
ends = []
for key in self._map_account_ids:
domain, mode = key
(domain, mode) = key
if mode == self.MODE_END and self.smart_end:
# postpone computation of ending balance
ends.append((domain, mode))
Expand All @@ -330,13 +330,16 @@ def do_queries(
domain.append(("account_id", "in", self._map_account_ids[key]))
if additional_move_line_filter:
domain.extend(additional_move_line_filter)

get_fields = ["debit", "credit", "account_id", "company_id"]
group_by_fields = ["account_id", "company_id"]
if auto_expand_col_name:
get_fields = [auto_expand_col_name] + get_fields
group_by_fields = [auto_expand_col_name] + group_by_fields

# fetch sum of debit/credit, grouped by account_id
accs = aml_model.read_group(
domain,
["debit", "credit", "account_id", "company_id"],
["account_id", "company_id"],
lazy=False,
)
accs = aml_model.read_group(domain, get_fields, group_by_fields, lazy=False)

for acc in accs:
rate, dp = company_rates[acc["company_id"][0]]
debit = acc["debit"] or 0.0
Expand All @@ -346,19 +349,45 @@ def do_queries(
):
# in initial mode, ignore accounts with 0 balance
continue
self._data[key][acc["account_id"][0]] = (debit * rate, credit * rate)
if (
auto_expand_col_name
and auto_expand_col_name in acc
and acc[auto_expand_col_name]
):
rdi_id = acc[auto_expand_col_name][0]
else:
rdi_id = UNCLASSIFIED_ROW_DETAIL
if not self._data[key].get(rdi_id, False):
self._data[key][rdi_id] = defaultdict(dict)
self._data[key][rdi_id][acc["account_id"][0]] = (
debit * rate,
credit * rate,
)
# compute ending balances by summing initial and variation
for key in ends:
domain, mode = key
initial_data = self._data[(domain, self.MODE_INITIAL)]
variation_data = self._data[(domain, self.MODE_VARIATION)]
account_ids = set(initial_data.keys()) | set(variation_data.keys())
for account_id in account_ids:
di, ci = initial_data.get(account_id, (AccountingNone, AccountingNone))
dv, cv = variation_data.get(
account_id, (AccountingNone, AccountingNone)
rdis = set(initial_data.keys()) | set(variation_data.keys())
for rdi in rdis:
if not initial_data.get(rdi, False):
initial_data[rdi] = defaultdict(dict)
if not variation_data.get(rdi, False):
variation_data[rdi] = defaultdict(dict)
if not self._data[key].get(rdi, False):
self._data[key][rdi] = defaultdict(dict)

account_ids = set(initial_data[rdi].keys()) | set(
variation_data[rdi].keys()
)
self._data[key][account_id] = (di + dv, ci + cv)
for account_id in account_ids:
di, ci = initial_data[rdi].get(
account_id, (AccountingNone, AccountingNone)
)
dv, cv = variation_data[rdi].get(
account_id, (AccountingNone, AccountingNone)
)
self._data[key][rdi][account_id] = (di + dv, ci + cv)

def replace_expr(self, expr):
"""Replace accounting variables in an expression by their amount.
Expand All @@ -371,23 +400,25 @@ def replace_expr(self, expr):
def f(mo):
field, mode, acc_domain, ml_domain = self._parse_match_object(mo)
key = (ml_domain, mode)
account_ids_data = self._data[key]
rdi_ids_data = self._data[key]
v = AccountingNone
account_ids = self._account_ids_by_acc_domain[acc_domain]
for account_id in account_ids:
debit, credit = account_ids_data.get(
account_id, (AccountingNone, AccountingNone)
)
if field == "bal":
v += debit - credit
elif field == "pbal" and debit >= credit:
v += debit - credit
elif field == "nbal" and debit < credit:
v += debit - credit
elif field == "deb":
v += debit
elif field == "crd":
v += credit
for rdi in rdi_ids_data:
account_ids_data = self._data[key][rdi]
for account_id in account_ids:
debit, credit = account_ids_data.get(
account_id, (AccountingNone, AccountingNone)
)
if field == "bal":
v += debit - credit
elif field == "pbal" and debit >= credit:
v += debit - credit
elif field == "nbal" and debit < credit:
v += debit - credit
elif field == "deb":
v += debit
elif field == "crd":
v += credit
# in initial balance mode, assume 0 is None
# as it does not make sense to distinguish 0 from "no data"
if (
Expand All @@ -401,11 +432,11 @@ def f(mo):
return self._ACC_RE.sub(f, expr)

def replace_exprs_by_account_id(self, exprs):
"""Replace accounting variables in a list of expression
by their amount, iterating by accounts involved in the expression.
"""This method is depreciated and replaced by replace_exprs_by_row_detail.
Replace accounting variables in a list of expression
by their amount, iterating by accounts involved in the expression.
yields account_id, replaced_expr
This method must be executed after do_queries().
"""

Expand All @@ -417,7 +448,7 @@ def f(mo):
if account_id not in self._account_ids_by_acc_domain[acc_domain]:
return "(AccountingNone)"
# here we know account_id is involved in acc_domain
account_ids_data = self._data[key]
account_ids_data = self._data[key][UNCLASSIFIED_ROW_DETAIL]
debit, credit = account_ids_data.get(
account_id, (AccountingNone, AccountingNone)
)
Expand Down Expand Up @@ -452,14 +483,66 @@ def f(mo):
for mo in self._ACC_RE.finditer(expr):
field, mode, acc_domain, ml_domain = self._parse_match_object(mo)
key = (ml_domain, mode)
account_ids_data = self._data[key]
account_ids_data = self._data[key][UNCLASSIFIED_ROW_DETAIL]
for account_id in self._account_ids_by_acc_domain[acc_domain]:
if account_id in account_ids_data:
account_ids.add(account_id)

for account_id in account_ids:
yield account_id, [self._ACC_RE.sub(f, expr) for expr in exprs]

def replace_exprs_by_row_detail(self, exprs):
"""Replace accounting variables in a list of expression
by their amount, iterating by accounts involved in the expression.
yields account_id, replaced_expr
This method must be executed after do_queries().
"""

def f(mo):
field, mode, acc_domain, ml_domain = self._parse_match_object(mo)
key = (ml_domain, mode)
v = AccountingNone
account_ids_data = self._data[key][rdi_id]
account_ids = self._account_ids_by_acc_domain[acc_domain]

for account_id in account_ids:
debit, credit = account_ids_data.get(
account_id, (AccountingNone, AccountingNone)
)
if field == "bal":
v += debit - credit
elif field == "pbal" and debit >= credit:
v += debit - credit
elif field == "nbal" and debit < credit:
v += debit - credit
elif field == "deb":
v += debit
elif field == "crd":
v += credit
# in initial balance mode, assume 0 is None
# as it does not make sense to distinguish 0 from "no data"
if (
v is not AccountingNone
and mode in (self.MODE_INITIAL, self.MODE_UNALLOCATED)
and float_is_zero(v, precision_digits=self.dp)
):
v = AccountingNone
return "(" + repr(v) + ")"

rdi_ids = set()
for expr in exprs:
for mo in self._ACC_RE.finditer(expr):
field, mode, acc_domain, ml_domain = self._parse_match_object(mo)
key = (ml_domain, mode)
rdis_data = self._data[key]
for rdi_id in rdis_data.keys():
rdi_ids.add(rdi_id)

for rdi_id in rdi_ids:
yield rdi_id, [self._ACC_RE.sub(f, expr) for expr in exprs]

@classmethod
def _get_balances(cls, mode, companies, date_from, date_to):
expr = "deb{mode}[], crd{mode}[]".format(mode=mode)
Expand All @@ -470,7 +553,10 @@ def _get_balances(cls, mode, companies, date_from, date_to):
aep.parse_expr(expr)
aep.done_parsing()
aep.do_queries(date_from, date_to)
return aep._data[((), mode)]

return aep._data[((), mode)].get(UNCLASSIFIED_ROW_DETAIL, {})
# to keep compatibility, we give the UNCLASSIFIED_ROW_DETAIL
# (expecting that auto_expand_col_names=None was given to do_queries )

@classmethod
def get_balances_initial(cls, companies, date):
Expand Down
21 changes: 20 additions & 1 deletion mis_builder/models/expression_evaluator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,14 @@ def __init__(
self.aml_model = aml_model
self._aep_queries_done = False

def aep_do_queries(self):
def aep_do_queries(self, auto_expand_col_name=None):
if self.aep and not self._aep_queries_done:
self.aep.do_queries(
self.date_from,
self.date_to,
self.additional_move_line_filter,
self.aml_model,
auto_expand_col_name,
)
self._aep_queries_done = True

Expand All @@ -50,6 +51,7 @@ def eval_expressions(self, expressions, locals_dict):
drilldown_args.append(None)
return vals, drilldown_args, name_error

# we keep it for backward compatibility
def eval_expressions_by_account(self, expressions, locals_dict):
if not self.aep:
return
Expand All @@ -66,3 +68,20 @@ def eval_expressions_by_account(self, expressions, locals_dict):
else:
drilldown_args.append(None)
yield account_id, vals, drilldown_args, name_error

def eval_expressions_by_row_detail(self, expressions, locals_dict):
if not self.aep:
return
exprs = [e and e.name or "AccountingNone" for e in expressions]
for rdi_id, replaced_exprs in self.aep.replace_exprs_by_row_detail(exprs):
vals = []
drilldown_args = []
name_error = False
for expr, replaced_expr in zip(exprs, replaced_exprs):
val = mis_safe_eval(replaced_expr, locals_dict)
vals.append(val)
if replaced_expr != expr:
drilldown_args.append({"expr": expr, "row_detail": rdi_id})
else:
drilldown_args.append(None)
yield rdi_id, vals, drilldown_args, name_error
Loading

0 comments on commit 3e21d12

Please sign in to comment.