merge from devel

radical-cybertools · May 3, 2024 · 4fdd0bc · 4fdd0bc
2 parents 25ae804 + 5a0f335
commit 4fdd0bc
Show file tree

Hide file tree

Showing 13 changed files with 273 additions and 67 deletions.
diff --git a/.coveragerc b/.coveragerc
diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml
@@ -36,13 +36,14 @@ jobs:
         LOC: testenv/lib/python3.7/site-packages
       run: |
         . testenv/bin/activate
+        radical-stack
         coverage run --include=$LOC/radical/entk/* -m pytest -ra --timeout=600 -vvv --showlocals tests/test_component/ tests/test_integration/ tests/test_utils/
-        coverage combine
         coverage xml
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - uses: codecov/codecov-action@v3
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       with:
-        file: ./coverage.xml
+        files: ./coverage.xml
 
   linting:
     runs-on: ubuntu-latest

diff --git a/examples/misc/darshan_enabled.py b/examples/misc/darshan_enabled.py
@@ -18,6 +18,14 @@
     'cpus'    : 32,
     'walltime': 15
 }
+# RESOURCE_DESCRIPTION = {
+#     # https://radicalpilot.readthedocs.io/en/stable/supported/frontier.html
+#     'resource': 'ornl.frontier',
+#     'project' : 'NNNMMM',
+#     'queue'   : 'batch',
+#     'cpus'    : 56,
+#     'walltime': 15
+# }
 
 os.environ['RADICAL_LOG_LVL'] = 'DEBUG'
 os.environ['RADICAL_REPORT']  = 'TRUE'
@@ -71,10 +79,14 @@ def get_stage_1():
 
 def main():
 
-    cache_darshan_env(darshan_runtime_root='$DARSHAN_RUNTIME_ROOT',
-                      modules=['e4s/22.08/PrgEnv-gnu',
-                               'darshan-runtime',
-                               'darshan-util'])
+    if 'polaris' in RESOURCE_DESCRIPTION['resource']:
+        cache_darshan_env(
+            darshan_runtime_root='/soft/perftools/darshan/darshan-3.4.4',
+            modules=['darshan/3.4.4'])
+    elif 'frontier' in RESOURCE_DESCRIPTION['resource']:
+        cache_darshan_env(
+            darshan_runtime_root='$OLCF_DARSHAN_RUNTIME_ROOT',
+            modules=['darshan-runtime/3.4.0', 'darshan-util/3.4.0'])
 
     pipeline = re.Pipeline()
     pipeline.add_stages([get_stage_0(), get_stage_1()])

diff --git a/requirements-tests.txt b/requirements-tests.txt
@@ -7,7 +7,6 @@ pylint
 flake8
 timeout-decorator
 coverage
-codecov
 hypothesis
 mock==2.0.0.
 psutil

diff --git a/src/radical/entk/appman/appmanager.py b/src/radical/entk/appman/appmanager.py
@@ -840,6 +840,9 @@ def _update_task(self, tdict):
                             self._logger.debug('Task %s rts_uid set to %s',
                                                task.uid, task.rts_uid)
 
+                        if completed_task.metadata:
+                            task.metadata.update(completed_task.metadata)
+
                         if task.state in [DONE, FAILED]:
                             self._logger.debug('No change on task state %s \
                                              in state %s', task.uid, task.state)

diff --git a/src/radical/entk/appman/wfprocessor.py b/src/radical/entk/appman/wfprocessor.py
@@ -345,6 +345,8 @@ def _update_dequeued_task(self, deq_task):
                             task.path = str(deq_task.path)
                         if not task.rts_uid and deq_task.rts_uid:
                             task.rts_uid = str(deq_task.rts_uid)
+                        if deq_task.metadata:
+                            task.metadata.update(deq_task.metadata)
 
                         # If there is no exit code, we assume success
                         # We are only concerned about state of task and not

diff --git a/src/radical/entk/execman/rp/task_processor.py b/src/radical/entk/execman/rp/task_processor.py
@@ -480,8 +480,9 @@ def create_td_from_task(task, placeholders, task_hash_table, pkl_path, sid,
         td.post_launch    = task.post_launch
         td.stage_on_error = task.stage_on_error
 
+        td.metadata       = task.metadata
         if task.annotations:
-            td.metadata   = {'data': task.annotations.as_dict()}
+            td.metadata.update(data=task.annotations.as_dict())
 
         if task.parent_pipeline['uid']:
             td.tags = resolve_tags(
@@ -559,7 +560,8 @@ def create_task_from_rp(rp_task, logger, prof=None):
                                 'name': task_info_tuple[3].strip()},
             'parent_pipeline': {'uid' : task_info_tuple[4].strip(),
                                 'name': task_info_tuple[5].strip()},
-            'rts_uid'                 : rp_task.uid
+            'rts_uid'                 : rp_task.uid,
+            'metadata'                : rp_task.description.get('metadata', {})
         })
 
         if   rp_task.state == rp.DONE                  : task.exit_code = 0

diff --git a/src/radical/entk/task.py b/src/radical/entk/task.py
@@ -384,6 +384,10 @@ class Task(ru.TypedDict):
         [type: `Annotations` | default: `None`] Annotations to describe task's
         input and output files, and sets dependencies between tasks.
 
+    .. data:: metadata
+
+        [type: `dict` | default: `{}`] User/system defined metadata.
+
     Read-only attributes
     --------------------
 
@@ -434,7 +438,8 @@ class Task(ru.TypedDict):
         'rts_uid'              : str,
         'parent_stage'         : {str: None},
         'parent_pipeline'      : {str: None},
-        'annotations'          : Annotations
+        'annotations'          : Annotations,
+        'metadata'             : {str: None}
     }
 
     # guaranteed attributes with default non-initialized values
@@ -473,7 +478,8 @@ class Task(ru.TypedDict):
         'tags'                 : None,
         'rts_uid'              : None,
         'parent_stage'         : {'uid': None, 'name': None},
-        'parent_pipeline'      : {'uid': None, 'name': None}
+        'parent_pipeline'      : {'uid': None, 'name': None},
+        'metadata'             : {}
     }
 
     # --------------------------------------------------------------------------