Merge branch 'main' into nams_scc_stack_lumi_new_scheduler

.github/workflows/tests.yml

@@ -63,6 +63,8 @@ jobs:
         with:
           flags: loki
           files: ./coverage.xml
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Run transformations tests
         run: |
@@ -74,6 +76,8 @@ jobs:
         with:
           flags: transformations
           files: ./coverage.xml
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Run lint_rules tests
         run: |
@@ -85,3 +89,5 @@ jobs:
         with:
           flags: lint_rules
           files: ./coverage.xml
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

cmake/loki_transform.cmake

@@ -225,10 +225,10 @@ function( loki_transform_target )
 
     set( options
          NO_PLAN_SOURCEDIR COPY_UNMODIFIED CPP CPP_PLAN INLINE_MEMBERS
-	 RESOLVE_SEQUENCE_ASSOCIATION DERIVE_ARGUMENT_ARRAY_SHAPE TRIM_VECTOR_SECTIONS
+	 RESOLVE_SEQUENCE_ASSOCIATION DERIVE_ARGUMENT_ARRAY_SHAPE TRIM_VECTOR_SECTIONS GLOBAL_VAR_OFFLOAD
     )
     set( single_value_args TARGET COMMAND MODE DIRECTIVE FRONTEND CONFIG PLAN )
-    set( multi_value_args SOURCES HEADERS DEFINITIONS )
+    set( multi_value_args SOURCES HEADERS DEFINITIONS INCLUDES )
 
     cmake_parse_arguments( _PAR_T "${options}" "${single_value_args}" "${multi_value_args}" ${ARGN} )
 
@@ -307,6 +307,10 @@ function( loki_transform_target )
             list( APPEND _TRANSFORM_OPTIONS TRIM_VECTOR_SECTIONS )
         endif()
 
+        if( _PAR_T_GLOBAL_VAR_OFFLOAD )
+            list( APPEND _TRANSFORM_OPTIONS GLOBAL_VAR_OFFLOAD )
+        endif()
+
         loki_transform(
             COMMAND     ${_PAR_T_COMMAND}
             OUTPUT      ${LOKI_SOURCES_TO_APPEND}
@@ -318,6 +322,7 @@ function( loki_transform_target )
             SOURCES     ${_PAR_T_SOURCES}
             HEADERS     ${_PAR_T_HEADERS}
             DEFINITIONS ${_PAR_T_DEFINITIONS}
+            INCLUDES    ${_PAR_T_INCLUDES}
             DEPENDS     ${LOKI_SOURCES_TO_TRANSFORM} ${_PAR_T_HEADER} ${_PAR_T_CONFIG}
             ${_TRANSFORM_OPTIONS}
         )

loki/backend/cgen.py

@@ -10,10 +10,13 @@
     PREC_UNARY, PREC_LOGICAL_OR, PREC_LOGICAL_AND, PREC_NONE, PREC_CALL
 )
 
+from loki.tools import as_tuple
 from loki.ir import Import, Stringifier, FindNodes
-from loki.expression import LokiStringifyMapper, Array, symbolic_op, Literal
+from loki.expression import (
+        LokiStringifyMapper, Array, symbolic_op, Literal,
+        symbols as sym
+)
 from loki.types import BasicType, SymbolAttributes, DerivedType
-
 __all__ = ['cgen', 'CCodegen', 'CCodeMapper']
 
 
@@ -364,6 +367,40 @@ def visit_TypeDef(self, o, **kwargs):
         self.depth -= 1
         return self.join_lines(header, decls, footer)
 
+    def visit_MultiConditional(self, o, **kwargs):
+        """
+        Format as
+          switch case (<expr>) {
+          case <value>:
+            ...body...
+          [case <value>:]
+            [...body...]
+          [default:]
+            [...body...]
+          }
+        """
+        header = self.format_line('switch (', self.visit(o.expr, **kwargs), ') {')
+        cases = []
+        end_cases = []
+        for value in o.values:
+            if any(isinstance(val, sym.RangeIndex) for val in value):
+                # TODO: in Fortran a case can be a range, which is not straight-forward
+                #  to translate/transfer to C
+                #  https://j3-fortran.org/doc/year/10/10-007.pdf#page=200
+                raise NotImplementedError
+            case = self.visit_all(as_tuple(value), **kwargs)
+            cases.append(self.format_line('case ', self.join_items(case), ':'))
+            end_cases.append(self.format_line('break;'))
+        if o.else_body:
+            cases.append(self.format_line('default: '))
+            end_cases.append(self.format_line('break;'))
+        footer = self.format_line('}')
+        self.depth += 1
+        bodies = self.visit_all(*o.bodies, o.else_body, **kwargs)
+        self.depth -= 1
+        branches = [item for branch in zip(cases, bodies, end_cases) for item in branch]
+        return self.join_lines(header, *branches, footer)
+
 
 def cgen(ir):
     """

loki/batch/scheduler.py

@@ -18,7 +18,7 @@
 )
 from loki.frontend import FP, REGEX, RegexParserClass
 from loki.tools import as_tuple, CaseInsensitiveDict, flatten
-from loki.logging import info, perf, warning, debug
+from loki.logging import info, perf, warning, debug, error
 
 
 __all__ = ['Scheduler']
@@ -377,6 +377,46 @@ def rekey_item_cache(self):
         )
 
     def process(self, transformation):
+        """
+        Process all :attr:`items` in the scheduler's graph with either
+        a :any:`Pipeline` or a single :any:`Transformation`.
+
+        A single :any:`Transformation` pass invokes
+        :meth:`process_transformation` individually, while a
+        :any:`Pipeline` will apply each contained transformation in
+        turn over the full dependency graph of the scheduler.
+
+        Parameters
+        ----------
+        transformation : :any:`Transformation` or :any:`Pipeline`
+            The transformation or transformation pipeline to apply
+        """
+        from loki.transform import Transformation, Pipeline  # pylint: disable=import-outside-toplevel
+
+        if isinstance(transformation, Transformation):
+            self.process_transformation(transformation=transformation)
+
+        elif isinstance(transformation, Pipeline):
+            self.process_pipeline(pipeline=transformation)
+
+        else:
+            error('[Loki::Scheduler] Batch processing requires Transformation or Pipeline object')
+            raise RuntimeError('[Loki] Could not batch process {transformation_or_pipeline}')
+
+    def process_pipeline(self, pipeline):
+        """
+        Process a given :any:`Pipeline` by applying its assocaited
+        transformations in turn.
+
+        Parameters
+        ----------
+        transformation : :any:`Pipeline`
+            The transformation pipeline to apply
+        """
+        for transformation in pipeline.transformations:
+            self.process_transformation(transformation)
+
+    def process_transformation(self, transformation):
         """
         Process all :attr:`items` in the scheduler's graph
 
@@ -396,6 +436,11 @@ def process(self, transformation):
         to ``True``. This uses the :attr:`filegraph` to process the dependency tree.
         If combined with a :any:`Transformation.item_filter`, only source files with
         at least one object corresponding to an item of that type are processed.
+
+        Parameters
+        ----------
+        transformation : :any:`Transformation`
+            The transformation to apply over the dependency tree
         """
         def _get_definition_items(_item, sgraph_items):
             # For backward-compatibility with the DependencyTransform and LinterTransformation

loki/transform/__init__.py

@@ -21,3 +21,4 @@
 from loki.transform.transform_extract_contained_procedures import * # noqa
 from loki.transform.transform_dead_code import * # noqa
 from loki.transform.transform_sanitise import * # noqa
+from loki.transform.pipeline import * # noqa

loki/transform/dependency_transform.py

@@ -8,7 +8,6 @@
 from pathlib import Path
 
 from loki.backend import fgen
-from loki.batch import SchedulerConfig
 from loki.expression import Variable, FindInlineCalls, SubstituteExpressions
 from loki.ir import (
     CallStatement, Import, Section, Interface, FindNodes, Transformer
@@ -250,6 +249,8 @@ def rename_calls(self, routine, targets=None, item=None):
             Optional list of subroutine names for which to modify the corresponding
             calls. If not provided, all calls are updated
         """
+        from loki.batch import SchedulerConfig  # pylint: disable=import-outside-toplevel,cyclic-import
+
         def _update_item(orig_name, new_name):
             # Update the ignore property if necessary
             if item and (matched_keys := SchedulerConfig.match_item_keys(orig_name, item.ignore)):
@@ -319,7 +320,7 @@ def rename_imports(self, source, imports, targets=None):
         for im in imports:
             if im.c_import:
                 target_symbol = im.module.split('.')[0].lower()
-                if targets and target_symbol.lower() in targets:
+                if targets and target_symbol.lower() in targets and 'intfb' in im.module.lower():
                     # Modify the the basename of the C-style header import
                     s = '.'.join(im.module.split('.')[1:])
                     im._update(module=f'{target_symbol}{self.suffix}.{s}')
@@ -468,6 +469,8 @@ def update_imports(self, source, imports, **kwargs):
         """
         Update imports of wrapped subroutines.
         """
+        from loki.batch import SchedulerConfig  # pylint: disable=import-outside-toplevel,cyclic-import
+
         targets = tuple(str(t).lower() for t in as_tuple(kwargs.get('targets')))
         if self.replace_ignore_items and (item := kwargs.get('item')):
             targets += tuple(str(i).lower() for i in item.ignore)
@@ -487,7 +490,7 @@ def _update_item(proc_name, module_name):
         for im in imports:
             if im.c_import:
                 target_symbol = im.module.split('.')[0].lower()
-                if targets and target_symbol.lower() in targets:
+                if targets and target_symbol.lower() in targets and 'intfb' in im.module.lower():
                     # Create a new module import with explicitly qualified symbol
                     modname = f'{target_symbol}{self.module_suffix}'
                     _update_item(target_symbol.lower(), modname)

loki/transform/pipeline.py

@@ -0,0 +1,74 @@
+# (C) Copyright 2018- ECMWF.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+from inspect import signature, Parameter
+
+
+class Pipeline:
+    """
+    A transformation pipeline that combines multiple :any:`Transformation`
+    passes and allows to apply them in unison.
+
+    The associated :any:`Transformation` objects are constructed from keyword
+    arguments in the constructor, so shared keywords get same initial value.
+
+    Attributes
+    ----------
+    transformations : list of :any:`Transformation`
+        The list of transformations applied to a source in this pipeline
+
+    Parameters
+    ----------
+    classes : tuple of types
+        A tuple of types from which to instantiate :any:`Transformation` objects.
+    *args : optional
+        Positional arguments that are passed on to the constructors of
+        all transformations
+    **kwargs : optional
+        Keyword arguments that are matched to the constructor
+        signature of the transformations.
+    """
+
+    def __init__(self, *args, classes=None, **kwargs):
+        self.transformations = []
+        for cls in classes:
+
+            # Get all relevant constructor parameters from teh MRO,
+            # but exclude catch-all keyword args, like ``**kwargs``
+            t_parameters = {
+                k: v for c in cls.__mro__ for k, v in signature(c).parameters.items()
+                if not v.kind == Parameter.VAR_KEYWORD
+            }
+            # Filter kwargs for this transformation class specifically
+            t_kwargs = {k: v for k, v in kwargs.items() if k in t_parameters}
+
+            # We need to apply our own default, if we are to honour inheritance
+            t_kwargs.update({
+                k: param.default for k, param in t_parameters.items()
+                if k not in t_kwargs and param.default is not None
+            })
+
+            # Then instantiate with the default *args and the derived **t_kwargs
+            self.transformations.append(cls(*args, **t_kwargs))
+
+    def apply(self, source, **kwargs):
+        """
+        Apply each associated :any:`Transformation` to :data:`source`
+
+        It dispatches to the respective :meth:`apply` of each
+        :any:`Transformation` in the order specified in the constructor.
+
+        Parameters
+        ----------
+        source : :any:`Sourcefile` or :any:`Module` or :any:`Subroutine`
+            The source item to transform.
+        **kwargs : optional
+            Keyword arguments that are passed on to the methods defining the
+            actual transformation.
+        """
+        for trafo in self.transformations:
+            trafo.apply(source, **kwargs)

loki/transform/transform_hoist_variables.py

@@ -101,12 +101,6 @@ class HoistVariablesAnalysis(Transformation):
     Traverses all subroutines to find the variables to be hoisted.
     Create a derived class and override :func:`find_variables<HoistVariablesAnalysis.find_variables>`
     to define which variables to be hoisted.
-
-    Parameters
-    ----------
-    key : str
-        Access identifier/key for the ``item.trafo_data`` dictionary. Only necessary to provide if several of
-        these transformations are carried out in succession.
     """
 
     _key = 'HoistVariablesTransformation'
@@ -116,10 +110,6 @@ class HoistVariablesAnalysis(Transformation):
 
     process_ignored_items = True
 
-    def __init__(self, key=None):
-        if key is not None:
-            self._key = key
-
     def transform_subroutine(self, routine, **kwargs):
         """
         Analysis applied to :any:`Subroutine` item.
@@ -197,18 +187,13 @@ class HoistVariablesTransformation(Transformation):
 
     Parameters
     ----------
-    key : str
-        Access identifier/key for the ``item.trafo_data`` dictionary. Only necessary to provide if several of
-        these transformations are carried out in succession.
     as_kwarguments : boolean
         Whether to pass the hoisted arguments as `args` or `kwargs`.
     """
 
     _key = 'HoistVariablesTransformation'
 
-    def __init__(self, key=None, as_kwarguments=False):
-        if key is not None:
-            self._key = key
+    def __init__(self, as_kwarguments=False):
         self.as_kwarguments = as_kwarguments
 
     def transform_subroutine(self, routine, **kwargs):
@@ -371,19 +356,16 @@ class HoistTemporaryArraysAnalysis(HoistVariablesAnalysis):
 
     Parameters
     ----------
-    key : str, optional
-        Access identifier/key for the ``item.trafo_data`` dictionary. Only necessary to provide if several of
-        these transformations are carried out in succession.
     dim_vars: tuple of str, optional
-        Variables to be within the dimensions of the arrays to be hoisted. If not provided, no checks will be done
-        for the array dimensions.
+        Variables to be within the dimensions of the arrays to be
+        hoisted. If not provided, no checks will be done for the array
+        dimensions.
     """
 
     # Apply in reverse order to recursively find all variables to be hoisted.
     reverse_traversal = True
 
-    def __init__(self, key=None, dim_vars=None, **kwargs):
-        super().__init__(key=key, **kwargs)
+    def __init__(self, dim_vars=None):
         self.dim_vars = dim_vars
         if self.dim_vars is not None:
             assert is_iterable(self.dim_vars)
@@ -414,17 +396,8 @@ class HoistTemporaryArraysTransformationAllocatable(HoistVariablesTransformation
     functionality/transformation, to hoist temporary arrays and make
     them ``allocatable``, including the actual *allocation* and
     *de-allocation*.
-
-    Parameters
-    ----------
-    key : str, optional
-        Access identifier/key for the ``item.trafo_data`` dictionary. Only necessary to provide if several of
-        these transformations are carried out in succession.
     """
 
-    def __init__(self, key=None, **kwargs):
-        super().__init__(key=key, **kwargs)
-
     def driver_variable_declaration(self, routine, variables):
         """
         Declares hoisted arrays as ``allocatable``, including *allocation* and *de-allocation*.