From dfc3a92a52f5c80f86f2ee5cbbe03b80b79b3f55 Mon Sep 17 00:00:00 2001 From: Michael Staneker Date: Thu, 4 Apr 2024 09:45:06 +0000 Subject: [PATCH 1/3] Loki string parser based on pymbolic parser Loki string parser: correct ordering of statements, further improved functionality and testing Load fortran intrinsic procedure names from FParser and expose via global var 'FORTRAN_INTRINSIC_PROCEDURES' Renaming, improving documentation, improving fortran style numerical literals, use FParser fortran intrinsic procedure names --- loki/analyse/tests/test_util_polyhedron.py | 23 +- loki/expression/__init__.py | 1 + loki/expression/parser.py | 438 ++++++++++++++++++++ loki/expression/tests/test_expression.py | 445 ++++++++++++++++++++- loki/expression/tests/test_symbolic.py | 28 +- loki/frontend/fparser.py | 8 +- loki/transform/transform_loop.py | 5 +- 7 files changed, 905 insertions(+), 43 deletions(-) create mode 100644 loki/expression/parser.py diff --git a/loki/analyse/tests/test_util_polyhedron.py b/loki/analyse/tests/test_util_polyhedron.py index 1642682ac..0c7931cf6 100644 --- a/loki/analyse/tests/test_util_polyhedron.py +++ b/loki/analyse/tests/test_util_polyhedron.py @@ -12,10 +12,8 @@ from loki.ir import Loop, FindNodes from loki.scope import Scope from loki.sourcefile import Sourcefile -from loki.frontend.fparser import parse_fparser_expression, HAVE_FP from loki.analyse.util_polyhedron import Polyhedron -from loki.expression import symbols as sym - +from loki.expression import symbols as sym, parse_expr @pytest.fixture(scope="module", name="here") def fixture_here(): @@ -27,9 +25,6 @@ def fixture_testdir(here): return here.parent.parent/'tests' -# Polyhedron functionality relies on FParser's expression parsing -pytestmark = pytest.mark.skipif(not HAVE_FP, reason="Fparser not available") - @pytest.mark.parametrize( "variables, lbounds, ubounds, A, b, variable_names", @@ -81,9 +76,9 @@ def test_polyhedron_from_loop_ranges(variables, lbounds, ubounds, A, b, variable Test converting loop ranges to polyedron representation of iteration space. """ scope = Scope() - loop_variables = [parse_fparser_expression(expr, scope) for expr in variables] - loop_lbounds = [parse_fparser_expression(expr, scope) for expr in lbounds] - loop_ubounds = [parse_fparser_expression(expr, scope) for expr in ubounds] + loop_variables = [parse_expr(expr, scope) for expr in variables] + loop_lbounds = [parse_expr(expr, scope) for expr in lbounds] + loop_ubounds = [parse_expr(expr, scope) for expr in ubounds] loop_ranges = [sym.LoopRange((l, u)) for l, u in zip(loop_lbounds, loop_ubounds)] p = Polyhedron.from_loop_ranges(loop_variables, loop_ranges) assert np.all(p.A == np.array(A, dtype=np.dtype(int))) @@ -97,15 +92,15 @@ def test_polyhedron_from_loop_ranges_failures(): """ # m*n is non-affine and thus can't be represented scope = Scope() - loop_variable = parse_fparser_expression("i", scope) - lower_bound = parse_fparser_expression("1", scope) - upper_bound = parse_fparser_expression("m * n", scope) + loop_variable = parse_expr("i", scope) + lower_bound = parse_expr("1", scope) + upper_bound = parse_expr("m * n", scope) loop_range = sym.LoopRange((lower_bound, upper_bound)) with pytest.raises(ValueError): _ = Polyhedron.from_loop_ranges([loop_variable], [loop_range]) # no functionality to flatten exponentials, yet - upper_bound = parse_fparser_expression("5**2", scope) + upper_bound = parse_expr("5**2", scope) loop_range = sym.LoopRange((lower_bound, upper_bound)) with pytest.raises(ValueError): _ = Polyhedron.from_loop_ranges([loop_variable], [loop_range]) @@ -148,7 +143,7 @@ def test_polyhedron_bounds(A, b, variable_names, lower_bounds, upper_bounds): Test the production of lower and upper bounds. """ scope = Scope() - variables = [parse_fparser_expression(v, scope) for v in variable_names] + variables = [parse_expr(v, scope) for v in variable_names] p = Polyhedron(A, b, variables) for var, ref_bounds in zip(variables, lower_bounds): lbounds = p.lower_bounds(var) diff --git a/loki/expression/__init__.py b/loki/expression/__init__.py index 1c08791f4..495515f2c 100644 --- a/loki/expression/__init__.py +++ b/loki/expression/__init__.py @@ -10,3 +10,4 @@ from loki.expression.operations import * # noqa from loki.expression.mappers import * # noqa from loki.expression.symbolic import * # noqa +from loki.expression.parser import * # noqa diff --git a/loki/expression/parser.py b/loki/expression/parser.py new file mode 100644 index 000000000..c180688b9 --- /dev/null +++ b/loki/expression/parser.py @@ -0,0 +1,438 @@ +# (C) Copyright 2018- ECMWF. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +from sys import intern +import re +import math +import pytools.lex +from pymbolic.parser import Parser as ParserBase # , FinalizedTuple +from pymbolic.mapper import Mapper +import pymbolic.primitives as pmbl +from pymbolic.mapper.evaluator import EvaluationMapper +from pymbolic.parser import ( + _openpar, _closepar, _minus, FinalizedTuple, _PREC_UNARY, + _PREC_TIMES, _PREC_PLUS, _times, _plus +) + +from loki.frontend.fparser import FORTRAN_INTRINSIC_PROCEDURES +from loki.tools.util import CaseInsensitiveDict +import loki.expression.symbols as sym +import loki.expression.operations as sym_ops +from loki.expression.expr_visitors import AttachScopes +from loki.scope import Scope + +__all__ = ['ExpressionParser', 'parse_expr'] + + +class PymbolicMapper(Mapper): + """ + Pymbolic expression to Loki expression mapper. + + Convert pymbolic expressions to Loki expressions. + """ + # pylint: disable=abstract-method,unused-argument + + def map_product(self, expr, *args, **kwargs): + children = tuple(self.rec(child, *args, **kwargs) for child in expr.children) + if isinstance(expr, sym_ops.ParenthesisedMul): + return sym_ops.ParenthesisedMul(children) + return sym.Product(children) + + def map_sum(self, expr, *args, **kwargs): + children = tuple(self.rec(child, *args, **kwargs) for child in expr.children) + if isinstance(expr, sym_ops.ParenthesisedAdd): + return sym_ops.ParenthesisedAdd(children) + return sym.Sum(children) + + def map_power(self, expr, *args, **kwargs): + base=self.rec(expr.base, *args, **kwargs) + exponent=self.rec(expr.exponent, *args, **kwargs) + if isinstance(expr, sym_ops.ParenthesisedPow): + return sym_ops.ParenthesisedPow(base=base, exponent=exponent) + return sym.Power(base=base, exponent=exponent) + + def map_quotient(self, expr, *args, **kwargs): + numerator=self.rec(expr.numerator, *args, **kwargs) + denominator=self.rec(expr.denominator, *args, **kwargs) + if isinstance(expr, sym_ops.ParenthesisedDiv): + return sym_ops.ParenthesisedDiv(numerator=numerator, denominator=denominator) + return sym.Quotient(numerator=numerator, denominator=denominator) + + def map_comparison(self, expr, *args, **kwargs): + return sym.Comparison(left=self.rec(expr.left, *args, **kwargs), + operator=expr.operator, + right=self.rec(expr.right, *args, **kwargs)) + + def map_logical_and(self, expr, *args, **kwargs): + return sym.LogicalAnd(tuple(self.rec(child, *args, **kwargs) for child in expr.children)) + + def map_logical_or(self, expr, *args, **kwargs): + return sym.LogicalOr(tuple(self.rec(child, *args, **kwargs) for child in expr.children)) + + def map_logical_not(self, expr, *args, **kwargs): + return sym.LogicalNot(self.rec(expr.child, *args, **kwargs)) + + def map_constant(self, expr, *args, **kwargs): + if expr == -1: + return expr + if isinstance(expr, (sym.FloatLiteral, sym.IntLiteral, sym.StringLiteral, sym.LogicLiteral)): + if isinstance(expr, sym.IntLiteral) and expr.value < 0: + return sym.Product((-1, sym.IntLiteral(abs(expr.value)))) + return expr + if isinstance(expr, bool): + return sym.LogicLiteral('true' if expr else 'false') + return sym.Literal(expr) + + map_logic_literal = map_constant + map_string_literal = map_constant + map_intrinsic_literal = map_constant + + map_int_literal = map_constant + + map_float_literal = map_int_literal + map_variable_symbol = map_constant + map_deferred_type_symbol = map_constant + + def map_meta_symbol(self, expr, *args, **kwargs): + return sym.Variable(name=str(expr.name)) + map_scalar = map_meta_symbol + map_array = map_meta_symbol + + def map_slice(self, expr, *args, **kwargs): + return sym.RangeIndex(tuple(self.rec(child, *args, **kwargs) for child in expr.children)) + + map_range = map_slice + map_range_index = map_slice + map_loop_range = map_slice + + def map_variable(self, expr, *args, **kwargs): + return sym.Variable(name=expr.name) + + def map_algebraic_leaf(self, expr, *args, **kwargs): + if str(expr).isnumeric(): + return self.map_constant(expr) + if isinstance(expr, pmbl.Call): + if expr.function.name.lower() in ('real', 'int'): + return sym.Cast(expr.function.name, [self.rec(param, *args, **kwargs) for param in expr.parameter][0]) + if expr.function.name.upper() in FORTRAN_INTRINSIC_PROCEDURES: + return sym.InlineCall(function=sym.Variable(name=expr.function.name), + parameters=tuple(self.rec(param, *args, **kwargs) for param in expr.parameters)) + return sym.Variable(name=expr.function.name, + dimensions=tuple(self.rec(param, *args, **kwargs) for param in expr.parameters)) + try: + return self.map_variable(expr, *args, **kwargs) + except Exception as e: + print(f"Exception: {e}") + return expr + + def map_call_with_kwargs(self, expr, *args, **kwargs): + name = sym.Variable(name=expr.function.name) + parameters = tuple(self.rec(param, *args, **kwargs) for param in expr.parameters) + kw_parameters = {key: self.rec(value, *args, **kwargs) for key, value\ + in CaseInsensitiveDict(expr.kw_parameters).items()} + if expr.function.name.lower() in ('real', 'int'): + return sym.Cast(name, parameters, kind=kw_parameters['kind']) + + return sym.InlineCall(function=name, parameters=parameters, kw_parameters=kw_parameters) + + def map_tuple(self, expr, *args, **kwargs): + return tuple(self.rec(elem, *args, **kwargs) for elem in expr) + + def map_list(self, expr, *args, **kwargs): + return sym.LiteralList([self.rec(elem, *args, **kwargs) for elem in expr]) + + +class LokiEvaluationMapper(EvaluationMapper): + """ + A mapper for evaluating expressions, based on + :any:`pymbolic.mapper.evaluator.EvaluationMapper`. + + Parameters + ---------- + strict : bool + Raise exception for unknown symbols/expressions (default: `False`). + """ + + def __init__(self, strict=False, **kwargs): + self.strict = strict + super().__init__(**kwargs) + + def map_logic_literal(self, expr): + return expr.value + + def map_float_literal(self, expr): + return expr.value + map_int_literal = map_float_literal + + def map_variable(self, expr): + if expr.name.upper() in FORTRAN_INTRINSIC_PROCEDURES: + return self.map_call(expr) + if self.strict: + return super().map_variable(expr) + if expr.name in self.context: + return super().map_variable(expr) + return expr + + def map_call(self, expr): + if expr.function.name.lower() == 'min': + return min(self.rec(par) for par in expr.parameters) + if expr.function.name.lower() == 'max': + return max(self.rec(par) for par in expr.parameters) + if expr.function.name.lower() == 'modulo': + args = [self.rec(par) for par in expr.parameters] + return args[0]%args[1] + if expr.function.name.lower() == 'abs': + return abs(float([self.rec(par) for par in expr.parameters][0])) + if expr.function.name.lower() == 'int': + return int(float([self.rec(par) for par in expr.parameters][0])) + if expr.function.name.lower() == 'real': + return float([self.rec(par) for par in expr.parameters][0]) + if expr.function.name.lower() == 'sqrt': + return math.sqrt(float([self.rec(par) for par in expr.parameters][0])) + if expr.function.name.lower() == 'exp': + return math.exp(float([self.rec(par) for par in expr.parameters][0])) + return super().map_call(expr) + + +class ExpressionParser(ParserBase): + """ + String Parser based on `pymbolic's `_ parser for + parsing expressions from strings. + + The Loki String Parser utilises and extends pymbolic's parser to incorporate + Fortran specific syntax and to map pymbolic expressions to Loki expressions, utilising + the mapper :any:`PymbolicMapper`. + + **Further**, in order to ensure correct ordering of Fortran Statements as documented + in `'WD 1539-1 J3/23-007r1 (Draft Fortran 2023)' `_, + pymbolic's parsing logic needed to be slightly adapted. + + Pymbolic references: + + * `GitHub: pymbolic `_ + * `pymbolic/parser.py `_ + * `pymbolic's parser documentation `_ + + .. note:: + **Example:** + Using the expression parser and possibly evaluate them + + .. code-block:: + + >>> from loki import parse_expr + >>> # parse numerical expressions + >>> ir = parse_expr('3 + 2**4') + >>> ir + Sum((IntLiteral(3, None), Power(IntLiteral(2, None), IntLiteral(4, None)))) + >>> # or expressions with variables + >>> ir = parse_expr('a*b') + >>> ir + Product((DeferredTypeSymbol('a', None, None, ),\ + DeferredTypeSymbol('b', None, None, ))) + >>> # and provide a scope e.g, with some routine defining a and b as 'real's + >>> ir = parse_expr('a*b', scope=routine) + >>> ir + Product((Scalar('a', None, None, None), Scalar('b', None, None, None))) + >>> # further, it is possible to evaluate expressions + >>> ir = parse_expr('a*b + 1', evaluate=True, context={'a': 2, 'b': 3}) + >>> ir + >>> IntLiteral(7, None) + >>> # even with functions implemented in Python + >>> def add(a, b): + >>> return a + b + >>> ir = parse_expr('a + add(a, b)', evaluate=True, context={'a': 2, 'b': 3, 'add': add}) + >>> ir + >>> IntLiteral(7, None) + """ + + _f_true = intern("f_true") + _f_false = intern("f_false") + _f_lessequal = intern('_f_lessequal') + _f_less = intern('_f_less') + _f_greaterequal = intern('_f_greaterequal') + _f_greater = intern('_f_greater') + _f_equal = intern('_f_equal') + _f_notequal = intern('_f_notequal') + _f_and = intern("and") + _f_or = intern("or") + _f_not = intern("not") + _f_float = intern("f_float") + _f_int = intern("f_int") + _f_string = intern("f_string") + _f_openbracket = intern("openbracket") + _f_closebracket = intern("closebracket") + + lex_table = [ + (_f_true, pytools.lex.RE(r"\.true\.", re.IGNORECASE)), + (_f_false, pytools.lex.RE(r"\.false\.", re.IGNORECASE)), + (_f_lessequal, pytools.lex.RE(r"\.le\.", re.IGNORECASE)), + (_f_less, pytools.lex.RE(r"\.lt\.", re.IGNORECASE)), + (_f_greaterequal, pytools.lex.RE(r"\.ge\.", re.IGNORECASE)), + (_f_greater, pytools.lex.RE(r"\.gt\.", re.IGNORECASE)), + (_f_equal, pytools.lex.RE(r"\.eq\.", re.IGNORECASE)), + (_f_notequal, pytools.lex.RE(r"\.ne\.", re.IGNORECASE)), + (_f_and, pytools.lex.RE(r"\.and\.", re.IGNORECASE)), + (_f_or, pytools.lex.RE(r"\.or\.", re.IGNORECASE)), + (_f_not, pytools.lex.RE(r"\.not\.", re.IGNORECASE)), + (_f_float, ("|", pytools.lex.RE(r"[0-9]+\.[0-9]*([eEdD][+-]?[0-9]+)?(_([\w$]+|[0-9]+))+$", re.IGNORECASE))), + (_f_int, pytools.lex.RE(r"[0-9]+?(_[a-zA-Z]*)", re.IGNORECASE)), + (_f_string, ("|", pytools.lex.RE(r'\".*\"', re.IGNORECASE), + pytools.lex.RE(r"\'.*\'", re.IGNORECASE))), + (_f_openbracket, pytools.lex.RE(r"\(/")), + (_f_closebracket, pytools.lex.RE(r"/\)")), + ] + ParserBase.lex_table + + ParserBase._COMP_TABLE.update({ + _f_lessequal: "<=", + _f_less: "<", + _f_greaterequal: ">=", + _f_greater: ">", + _f_equal: "==", + _f_notequal: "!=" + }) + + @staticmethod + def _parenthesise(expr): + if isinstance(expr, pmbl.Sum): + return sym_ops.ParenthesisedAdd(expr.children) + if isinstance(expr, pmbl.Product): + return sym_ops.ParenthesisedMul(expr.children) + if isinstance(expr, pmbl.Quotient): + return sym_ops.ParenthesisedDiv(numerator=expr.numerator, + denominator=expr.denominator) + if isinstance(expr, pmbl.Power): + return sym_ops.ParenthesisedPow(base=expr.base, exponent=expr.exponent) + return expr + + def parse_prefix(self, pstate): + pstate.expect_not_end() + + if pstate.is_next(_minus): + pstate.advance() + left_exp = pmbl.Product((-1, self.parse_expression(pstate, _PREC_UNARY))) + return left_exp + if pstate.is_next(_openpar): + pstate.advance() + + if pstate.is_next(_closepar): + left_exp = () + else: + # This is parsing expressions separated by commas, so it + # will return a tuple. Kind of the lazy way out. + left_exp = self.parse_expression(pstate) + # NECESSARY to ensure correct ordering! + left_exp = self._parenthesise(left_exp) + pstate.expect(_closepar) + pstate.advance() + if isinstance(left_exp, tuple): + # These could just be plain parentheses. + + # Finalization prevents things from being appended + # to containers after their closing delimiter. + left_exp = FinalizedTuple(left_exp) + return left_exp + return super().parse_prefix(pstate) + + def parse_postfix(self, pstate, min_precedence, left_exp): + + did_something = False + if pstate.is_next(_times) and _PREC_TIMES > min_precedence: + pstate.advance() + right_exp = self.parse_expression(pstate, _PREC_PLUS) + # NECESSARY to ensure correct ordering! + # pylint: disable=unidiomatic-typecheck + if type(right_exp) is pmbl.Quotient: + left_exp = pmbl.Quotient(numerator=pmbl.Product((left_exp, right_exp.numerator)), + denominator=right_exp.denominator) + # pylint: disable=unidiomatic-typecheck + elif type(right_exp) is pmbl.Product: + left_exp = pmbl.Product((sym.Product((left_exp, right_exp.children[0])), right_exp.children[1])) + else: + left_exp = pmbl.Product((left_exp, right_exp)) + did_something = True + elif pstate.is_next(_plus) and _PREC_PLUS > min_precedence: + pstate.advance() + right_exp = self.parse_expression(pstate, _PREC_PLUS) + left_exp = pmbl.Sum((left_exp, right_exp)) + did_something = True + elif pstate.is_next(_minus) and _PREC_PLUS > min_precedence: + pstate.advance() + right_exp = self.parse_expression(pstate, _PREC_PLUS) + right_exp = pmbl.Product((-1, right_exp)) + left_exp = pmbl.Sum((left_exp, right_exp)) + did_something = True + else: + return super().parse_postfix(pstate, min_precedence, left_exp) + return left_exp, did_something + + def parse_terminal(self, pstate): + if pstate.is_next(self._f_float): + return self.parse_f_float(pstate.next_str_and_advance()) + if pstate.is_next(self._f_int): + return self.parse_f_int(pstate.next_str_and_advance()) + if pstate.is_next(self._f_string): + return self.parse_f_string(pstate.next_str_and_advance()) + if pstate.is_next(self._f_true): + assert pstate.next_str_and_advance().lower() == ".true." + return sym.LogicLiteral('.TRUE.') + if pstate.is_next(self._f_false): + assert pstate.next_str_and_advance().lower() == ".false." + return sym.LogicLiteral('.FALSE.') + return super().parse_terminal(pstate) + + def __call__(self, expr_str, scope=None, evaluate=False, strict=False, context=None): + """ + Call Loki String Parser to convert expression(s) represented in a string to Loki expression(s)/IR. + + Parameters + ---------- + expr_str : str + The expression as a string + scope : :any:`Scope` + The scope to which symbol names inside the expression belong + evaluate : bool, optional + Whether to evaluate the expression or not (default: `False`) + strict : bool, optional + Whether to raise exception for unknown variables/symbols when + evaluating an expression (default: `False`) + context : dict, optional + Symbol context, defining variables/symbols/procedures to help/support + evaluating an expression + + Returns + ------- + :any:`Expression` + The expression tree corresponding to the expression + """ + result = super().__call__(expr_str) + context = context or {} + context = CaseInsensitiveDict(context) + if evaluate: + result = LokiEvaluationMapper(context=context, strict=strict)(result) + ir = PymbolicMapper()(result) + return AttachScopes().visit(ir, scope=scope or Scope()) + + def parse_f_float(self, s): + stripped = s.split('_', 1) + if len(stripped) == 2: + return sym.Literal(value=self.parse_float(stripped[0]), kind=sym.Variable(name=stripped[1].lower())) + return self.parse_float(stripped[0]) + + def parse_f_int(self, s): + stripped = s.split('_', 1) + value = int(stripped[0].replace("d", "e").replace("D", "e")) + return sym.IntLiteral(value=value, kind=sym.Variable(name=stripped[1].lower())) + + def parse_f_string(self, s): + return sym.StringLiteral(s) + + +parse_expr = ExpressionParser() +""" +An instance of :any:`ExpressionParser` that allows parsing expression strings into a Loki expression tree. +See :any:`ExpressionParser.__call__` for a description of the available arguments. +""" diff --git a/loki/expression/tests/test_expression.py b/loki/expression/tests/test_expression.py index c9492e0db..82da2f60d 100644 --- a/loki/expression/tests/test_expression.py +++ b/loki/expression/tests/test_expression.py @@ -22,7 +22,7 @@ from loki.build import jit_compile, clean_test from loki.expression import ( symbols as sym, FindVariables, FindExpressions, FindTypedSymbols, - FindInlineCalls, SubstituteExpressions, AttachScopesMapper + FindInlineCalls, SubstituteExpressions, AttachScopesMapper, parse_expr ) from loki.frontend import ( available_frontends, OFP, OMNI, FP, HAVE_FP, parse_fparser_expression @@ -1001,7 +1001,6 @@ def test_string_compare(): assert sym.LogicLiteral(value=True) == 'true' -@pytest.mark.skipif(not HAVE_FP, reason='Fparser not available') @pytest.mark.parametrize('expr, string, ref', [ ('a + 1', 'a', True), ('u(a)', 'a', True), @@ -1010,17 +1009,21 @@ def test_string_compare(): ('ansatz(a + 1)', 'a', True), ('ansatz(b + 1)', 'a', False), # Ensure no false positives ]) -def test_subexpression_match(expr, string, ref): +@pytest.mark.parametrize('parse', (parse_expr, parse_fparser_expression)) +def test_subexpression_match(parse, expr, string, ref): """ Test that we can identify individual symbols or sub-expressions in expressions via canonical string matching. """ scope = Scope() - expr = parse_fparser_expression(expr, scope) + if parse is parse_fparser_expression and not HAVE_FP: + with pytest.raises(RuntimeError): + expr = parse(expr, scope) + else: + expr = parse(expr, scope) assert (string in expr) == ref -@pytest.mark.skipif(not HAVE_FP, reason='Fparser not available') @pytest.mark.parametrize('source, ref', [ ('1 + 1', '1 + 1'), ('1+2+3+4', '1 + 2 + 3 + 4'), @@ -1030,12 +1033,17 @@ def test_subexpression_match(expr, string, ref): ('5 + (4 + 3) - (2*1)', '5 + (4 + 3) - (2*1)'), ('a*(b*(c+(d+e)))', 'a*(b*(c + (d + e)))'), ]) -def test_parse_fparser_expression(source, ref): +@pytest.mark.parametrize('parse', (parse_expr, parse_fparser_expression)) +def test_parse_expression(parse, source, ref): """ Test the utility function that parses simple expressions. """ scope = Scope() - ir = parse_fparser_expression(source, scope) + if parse is parse_fparser_expression and not HAVE_FP: + with pytest.raises(RuntimeError): + ir = parse(source, scope) + else: + ir = parse(source, scope) assert isinstance(ir, pmbl.Expression) assert str(ir) == ref @@ -1653,3 +1661,426 @@ def test_typebound_resolution_type_info(frontend): assert var.scope is sub assert isinstance(var, sym.DeferredTypeSymbol) assert var.type.dtype == dtype + + +# utility function to test parse_expr with different case +def convert_to_case(_str, mode='upper'): + if mode == 'upper': + return _str.upper() + if mode == 'lower': + return _str.lower() + if mode == 'random': + # this is obviously not random, but fulfils its purpose ... + result = '' + for i, char in enumerate(_str): + result += char.upper() if i%2==0 and i<3 else char.lower() + return result + return convert_to_case(_str) + + +@pytest.mark.parametrize('case', ('upper', 'lower', 'random')) +@pytest.mark.parametrize('frontend', available_frontends()) +def test_expression_parser(frontend, case): + fcode = """ +subroutine some_routine() + implicit none + integer :: i1, i2, i3, len1, len2, len3 + real :: a, b + real :: arr(len1, len2, len3) +end subroutine some_routine + """.strip() + + fcode_mod = """ +module external_mod + implicit none +contains + function my_func(a) + integer, intent(in) :: a + integer :: my_func + my_func = a + end function my_func +end module external_mod + """.strip() + + def to_str(_parsed): + return str(_parsed).lower().replace(' ', '') + + routine = Subroutine.from_source(fcode, frontend=frontend) + module = Module.from_source(fcode_mod, frontend=frontend) + + parsed = parse_expr(convert_to_case('a + b', mode=case)) + assert isinstance(parsed, sym.Sum) + assert all(isinstance(_parsed, sym.DeferredTypeSymbol) for _parsed in parsed.children) + assert to_str(parsed) == 'a+b' + + parsed = parse_expr(convert_to_case('a + b', mode=case), scope=routine) + assert isinstance(parsed, sym.Sum) + assert all(isinstance(_parsed, sym.Scalar) for _parsed in parsed.children) + assert all(_parsed.scope == routine for _parsed in parsed.children) + assert to_str(parsed) == 'a+b' + + parsed = parse_expr(convert_to_case('a + b + 2 + 10', mode=case), scope=routine) + assert isinstance(parsed, sym.Sum) + assert to_str(parsed) == 'a+b+2+10' + + parsed = parse_expr(convert_to_case('a - b', mode=case), scope=routine) + assert isinstance(parsed, sym.Sum) + assert isinstance(parsed.children[0], sym.Scalar) + assert isinstance(parsed.children[1], sym.Product) + assert to_str(parsed) == 'a-b' + + parsed = parse_expr(convert_to_case('a * b', mode=case), scope=routine) + assert isinstance(parsed, sym.Product) + assert all(isinstance(_parsed, sym.Scalar) for _parsed in parsed.children) + assert all(_parsed.scope == routine for _parsed in parsed.children) + assert to_str(parsed) == 'a*b' + + parsed = parse_expr(convert_to_case('a / b', mode=case), scope=routine) + assert isinstance(parsed, sym.Quotient) + assert all(isinstance(_parsed, sym.Scalar) for _parsed in [parsed.numerator, parsed.denominator]) + assert all(_parsed.scope == routine for _parsed in [parsed.numerator, parsed.denominator]) + assert to_str(parsed) == 'a/b' + + parsed = parse_expr(convert_to_case('a ** b', mode=case), scope=routine) + assert isinstance(parsed, sym.Power) + assert all(isinstance(_parsed, sym.Scalar) for _parsed in [parsed.base, parsed.exponent]) + assert all(_parsed.scope == routine for _parsed in [parsed.base, parsed.exponent]) + assert to_str(parsed) == 'a**b' + + parsed = parse_expr(convert_to_case('a:b', mode=case), scope=routine) + assert isinstance(parsed, sym.RangeIndex) + assert all(isinstance(_parsed, sym.Scalar) for _parsed in [parsed.lower, parsed.upper]) + assert all(_parsed.scope == routine for _parsed in [parsed.lower, parsed.upper]) + assert to_str(parsed) == 'a:b' + + parsed = parse_expr(convert_to_case('a:b:5', mode=case), scope=routine) + assert isinstance(parsed, sym.RangeIndex) + assert all(isinstance(_parsed, (sym.Scalar, sym.IntLiteral)) + for _parsed in [parsed.lower, parsed.upper, parsed.step]) + assert to_str(parsed) == 'a:b:5' + + parsed = parse_expr(convert_to_case('a == b', mode=case), scope=routine) + assert parsed.operator == '==' + assert isinstance(parsed, sym.Comparison) + assert all(isinstance(_parsed, sym.Scalar) for _parsed in [parsed.left, parsed.right]) + assert all(_parsed.scope == routine for _parsed in [parsed.left, parsed.right]) + assert to_str(parsed) == 'a==b' + parsed = parse_expr(convert_to_case('a.eq.b', mode=case), scope=routine) + assert parsed.operator == '==' + assert isinstance(parsed, sym.Comparison) + assert all(isinstance(_parsed, sym.Scalar) for _parsed in [parsed.left, parsed.right]) + assert all(_parsed.scope == routine for _parsed in [parsed.left, parsed.right]) + assert to_str(parsed) == 'a==b' + + parsed = parse_expr(convert_to_case('a!=b', mode=case), scope=routine) + assert parsed.operator == '!=' + assert isinstance(parsed, sym.Comparison) + assert all(isinstance(_parsed, sym.Scalar) for _parsed in [parsed.left, parsed.right]) + assert all(_parsed.scope == routine for _parsed in [parsed.left, parsed.right]) + assert to_str(parsed) == 'a!=b' + parsed = parse_expr(convert_to_case('a.ne.b', mode=case), scope=routine) + assert parsed.operator == '!=' + assert isinstance(parsed, sym.Comparison) + assert all(isinstance(_parsed, sym.Scalar) for _parsed in [parsed.left, parsed.right]) + assert all(_parsed.scope == routine for _parsed in [parsed.left, parsed.right]) + assert to_str(parsed) == 'a!=b' + + parsed = parse_expr(convert_to_case('a>b', mode=case), scope=routine) + assert parsed.operator == '>' + assert isinstance(parsed, sym.Comparison) + assert all(isinstance(_parsed, sym.Scalar) for _parsed in [parsed.left, parsed.right]) + assert all(_parsed.scope == routine for _parsed in [parsed.left, parsed.right]) + assert to_str(parsed) == 'a>b' + parsed = parse_expr(convert_to_case('a.gt.b', mode=case), scope=routine) + assert parsed.operator == '>' + assert isinstance(parsed, sym.Comparison) + assert all(isinstance(_parsed, sym.Scalar) for _parsed in [parsed.left, parsed.right]) + assert all(_parsed.scope == routine for _parsed in [parsed.left, parsed.right]) + assert to_str(parsed) == 'a>b' + + parsed = parse_expr(convert_to_case('a>=b', mode=case), scope=routine) + assert parsed.operator == '>=' + assert isinstance(parsed, sym.Comparison) + assert all(isinstance(_parsed, sym.Scalar) for _parsed in [parsed.left, parsed.right]) + assert all(_parsed.scope == routine for _parsed in [parsed.left, parsed.right]) + assert to_str(parsed) == 'a>=b' + parsed = parse_expr(convert_to_case('a.ge.b', mode=case), scope=routine) + assert parsed.operator == '>=' + assert isinstance(parsed, sym.Comparison) + assert all(isinstance(_parsed, sym.Scalar) for _parsed in [parsed.left, parsed.right]) + assert all(_parsed.scope == routine for _parsed in [parsed.left, parsed.right]) + assert to_str(parsed) == 'a>=b' + + parsed = parse_expr(convert_to_case('a Date: Wed, 17 Apr 2024 06:47:49 +0000 Subject: [PATCH 2/3] fix cyclic import, move 'FORTRAN_INTRINSIC_PROCEDURES' to expression/parser.py --- loki/expression/parser.py | 10 ++++++++-- loki/frontend/fparser.py | 8 ++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/loki/expression/parser.py b/loki/expression/parser.py index c180688b9..58c7581cf 100644 --- a/loki/expression/parser.py +++ b/loki/expression/parser.py @@ -17,15 +17,21 @@ _openpar, _closepar, _minus, FinalizedTuple, _PREC_UNARY, _PREC_TIMES, _PREC_PLUS, _times, _plus ) +try: + from fparser.two.Fortran2003 import Intrinsic_Name + + FORTRAN_INTRINSIC_PROCEDURES = Intrinsic_Name.function_names + """list of intrinsic fortran procedure(s) names""" +except ImportError: + FORTRAN_INTRINSIC_PROCEDURES = () -from loki.frontend.fparser import FORTRAN_INTRINSIC_PROCEDURES from loki.tools.util import CaseInsensitiveDict import loki.expression.symbols as sym import loki.expression.operations as sym_ops from loki.expression.expr_visitors import AttachScopes from loki.scope import Scope -__all__ = ['ExpressionParser', 'parse_expr'] +__all__ = ['ExpressionParser', 'parse_expr', 'FORTRAN_INTRINSIC_PROCEDURES'] class PymbolicMapper(Mapper): diff --git a/loki/frontend/fparser.py b/loki/frontend/fparser.py index 5cf1f60d3..4821eec93 100644 --- a/loki/frontend/fparser.py +++ b/loki/frontend/fparser.py @@ -15,14 +15,10 @@ from fparser.two.utils import get_child, walk, BlockBase from fparser.two import Fortran2003 from fparser.common.readfortran import FortranStringReader - from fparser.two.Fortran2003 import Intrinsic_Name - FORTRAN_INTRINSIC_PROCEDURES = Intrinsic_Name.function_names - """list of intrinsic fortran procedure(s) names""" HAVE_FP = True """Indicate whether fparser frontend is available.""" except ImportError: - FORTRAN_INTRINSIC_PROCEDURES = () HAVE_FP = False from loki.frontend.source import Source @@ -48,8 +44,8 @@ from loki.config import config -__all__ = ['HAVE_FP', 'FORTRAN_INTRINSIC_PROCEDURES', 'FParser2IR', 'parse_fparser_file', - 'parse_fparser_source', 'parse_fparser_ast', 'parse_fparser_expression', 'get_fparser_node'] +__all__ = ['HAVE_FP', 'FParser2IR', 'parse_fparser_file', 'parse_fparser_source', + 'parse_fparser_ast', 'parse_fparser_expression', 'get_fparser_node'] @Timer(logger=debug, text=lambda s: f'[Loki::FP] Executed parse_fparser_file in {s:.2f}s') From 44bccfd2a4fd0a977adebeb154e83c32f525d782 Mon Sep 17 00:00:00 2001 From: Michael Staneker Date: Wed, 17 Apr 2024 06:54:04 +0000 Subject: [PATCH 3/3] improve 'ExpressionParser's documentation --- loki/expression/parser.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/loki/expression/parser.py b/loki/expression/parser.py index 58c7581cf..b8a7a3f55 100644 --- a/loki/expression/parser.py +++ b/loki/expression/parser.py @@ -206,7 +206,7 @@ def map_call(self, expr): class ExpressionParser(ParserBase): """ - String Parser based on `pymbolic's `_ parser for + String Parser based on :any:`pymbolic.parser.Parser` for parsing expressions from strings. The Loki String Parser utilises and extends pymbolic's parser to incorporate @@ -253,6 +253,8 @@ class ExpressionParser(ParserBase): >>> ir = parse_expr('a + add(a, b)', evaluate=True, context={'a': 2, 'b': 3, 'add': add}) >>> ir >>> IntLiteral(7, None) + + .. automethod:: __call__ """ _f_true = intern("f_true") @@ -291,6 +293,9 @@ class ExpressionParser(ParserBase): (_f_openbracket, pytools.lex.RE(r"\(/")), (_f_closebracket, pytools.lex.RE(r"/\)")), ] + ParserBase.lex_table + """ + Extend :any:`pymbolic.parser.Parser.lex_table` to accomodate for Fortran specifix syntax/expressions. + """ ParserBase._COMP_TABLE.update({ _f_lessequal: "<=", @@ -303,6 +308,12 @@ class ExpressionParser(ParserBase): @staticmethod def _parenthesise(expr): + """ + Utility method to parenthesise specific expressions. + + E.g., from :any:`pymbolic.primitives.Sum` to + :any:`ParenthesisedAdd`. + """ if isinstance(expr, pmbl.Sum): return sym_ops.ParenthesisedAdd(expr.children) if isinstance(expr, pmbl.Product): @@ -423,17 +434,30 @@ def __call__(self, expr_str, scope=None, evaluate=False, strict=False, context=N return AttachScopes().visit(ir, scope=scope or Scope()) def parse_f_float(self, s): + """ + Parse "Fortran-style" float literals. + + E.g., ``3.1415_my_real_kind``. + """ stripped = s.split('_', 1) if len(stripped) == 2: return sym.Literal(value=self.parse_float(stripped[0]), kind=sym.Variable(name=stripped[1].lower())) return self.parse_float(stripped[0]) def parse_f_int(self, s): + """ + Parse "Fortran-style" int literals. + + E.g., ``1_my_int_kind``. + """ stripped = s.split('_', 1) value = int(stripped[0].replace("d", "e").replace("D", "e")) return sym.IntLiteral(value=value, kind=sym.Variable(name=stripped[1].lower())) def parse_f_string(self, s): + """ + Parse string literals. + """ return sym.StringLiteral(s)