From 69c5b3639cdb583f5afcab7489c98bebedeefaf7 Mon Sep 17 00:00:00 2001 From: Adrien Faure Date: Thu, 21 Dec 2023 11:59:27 +0100 Subject: [PATCH] [cli/api] detailed gives job types --- oar/api/routers/job.py | 7 +++++++ oar/cli/oarstat.py | 11 +++++++++++ oar/lib/basequery.py | 24 ++++++++++++++++++++++++ oar/lib/job_handling.py | 19 +++++++++++++------ tests/api/test_job.py | 33 ++++++++++++++++++++++++++++++++- tests/cli/test_oardel.py | 7 +++++-- tests/cli/test_oarstat.py | 25 +++++++++++++++++++++++++ 7 files changed, 117 insertions(+), 9 deletions(-) diff --git a/oar/api/routers/job.py b/oar/api/routers/job.py index d9d23f51..813edf5a 100644 --- a/oar/api/routers/job.py +++ b/oar/api/routers/job.py @@ -25,6 +25,11 @@ ) +def attach_types(job, job_types): + if job["id"] in job_types: + job["types"] = job_types[job["id"]] + + def attach_resources(job, jobs_resources): job["resources"] = [] for resource in jobs_resources[job["id"]]: @@ -75,9 +80,11 @@ def index( if details: jobs_resources = queryCollection.get_assigned_jobs_resources(page.items) + jobs_types = queryCollection.get_jobs_types(page.items) pass for item in page: if details: + attach_types(item, jobs_types) attach_resources(item, jobs_resources) attach_nodes(item, jobs_resources) data["items"].append(item) diff --git a/oar/cli/oarstat.py b/oar/cli/oarstat.py index be498214..059b3aaa 100644 --- a/oar/cli/oarstat.py +++ b/oar/cli/oarstat.py @@ -218,6 +218,17 @@ def print_jobs( ) job.network_adresses = nodes + jobs_types = queryCollection.get_jobs_types(jobs) + for job in jobs: + if job.id in jobs_types: + types = [] + for job_type, value in jobs_types[job.id].items(): + if type(value) == bool: + types.append(f"{job_type}") + else: + types.append(f"{job_type}={value}") + job.types = ", ".join(types) + if format: to_dump = {} # to_dict() doesn't incorporate attributes not defined in the class, thus the dict merging diff --git a/oar/lib/basequery.py b/oar/lib/basequery.py index 3415dbec..34103b44 100644 --- a/oar/lib/basequery.py +++ b/oar/lib/basequery.py @@ -2,6 +2,8 @@ from sqlalchemy import text from sqlalchemy.orm import Load +from oar.lib.job_handling import parse_job_type + # from . import db from .exceptions import DoesNotExist from .models import ( @@ -9,6 +11,7 @@ GanttJobsPredictionsVisu, GanttJobsResourcesVisu, Job, + JobType, MoldableJobDescription, Resource, ) @@ -214,6 +217,27 @@ def get_assigned_jobs_resources(self, jobs): ) return self.groupby_jobs_resources(jobs, query) + def get_jobs_types(self, jobs): + """Returns the list of assigned resources associated to the job passed + in parameter.""" + db = self.session + + results = ( + db.query(JobType) + .filter(JobType.job_id.in_([job.id for job in jobs])) + .order_by(JobType.job_id) + .all() + ) + + from itertools import groupby + + res = {} + for job_id, group_types in groupby(results, lambda type: type.job_id): + types_map = parse_job_type([j.type for j in list(group_types)]) + res[job_id] = types_map + + return res + def get_assigned_one_job_resources(self, job): """Returns the list of assigned resources associated to the job passed in parameter.""" diff --git a/oar/lib/job_handling.py b/oar/lib/job_handling.py index 2f5b46a2..1e6a5848 100644 --- a/oar/lib/job_handling.py +++ b/oar/lib/job_handling.py @@ -4,6 +4,7 @@ import os import random import re +from typing import List from procset import ProcSet from sqlalchemy import distinct, func, insert, text @@ -1572,13 +1573,9 @@ def get_job_current_hostnames(session, job_id): return [r[0] for r in results] -def get_job_types(session, job_id): - """Returns a hash table with all types for the given job ID.""" - - results = session.query(JobType.type).filter(JobType.job_id == job_id).all() +def parse_job_type(job_types: List[str]): res = {} - for t in results: - typ = t[0] + for typ in job_types: match = re.match(r"^\s*(token)\s*\:\s*(\w+)\s*=\s*(\d+)\s*$", typ) if match: res[match.group(1)] = {match.group(2): match.group(3)} @@ -1591,6 +1588,16 @@ def get_job_types(session, job_id): return res +def get_job_types(session, job_id): + """Returns a hash table with all types for the given job ID.""" + + results = session.query(JobType.type).filter(JobType.job_id == job_id).all() + # Pay attention to the comma that unpack the tuple of one element return by the request + res = parse_job_type([t for t, in results]) + + return res + + def add_current_job_types(session, job_id, j_type): req = insert(JobType).values({"job_id": job_id, "type": j_type}) session.execute(req) diff --git a/tests/api/test_job.py b/tests/api/test_job.py index 5d9c9d82..842ba6af 100644 --- a/tests/api/test_job.py +++ b/tests/api/test_job.py @@ -24,7 +24,16 @@ def test_jobs_get_all(client, minimal_db_initialization): def test_app_jobs_details(client, minimal_db_initialization): insert_job( - minimal_db_initialization, res=[(60, [("resource_id=4", "")])], properties="" + minimal_db_initialization, + res=[(60, [("resource_id=4", "")])], + properties="", + types=["be", "test=value"], + ) + insert_job( + minimal_db_initialization, + res=[(60, [("resource_id=4", "")])], + properties="", + types=["cosystem"], ) res = client.get("/jobs?details=details") @@ -676,3 +685,25 @@ def test_app_job_post_ar( assert job_ids == [] assert res.status_code == 403 + + +def test_app_jobs_details_check_types(client, minimal_db_initialization): + insert_job( + minimal_db_initialization, + res=[(60, [("resource_id=4", "")])], + properties="", + types=["be", "test=value"], + ) + insert_job( + minimal_db_initialization, + res=[(60, [("resource_id=4", "")])], + properties="", + types=["cosystem"], + ) + res = client.get("/jobs?details=details") + + print(res.json(), len(res.json())) + + assert res.json()["items"][0]["types"]["test"] == "value" + assert res.json()["items"][0]["types"]["be"] + assert res.json()["items"][1]["types"]["cosystem"] diff --git a/tests/cli/test_oardel.py b/tests/cli/test_oardel.py index 4fba623f..64e1f33d 100644 --- a/tests/cli/test_oardel.py +++ b/tests/cli/test_oardel.py @@ -79,12 +79,15 @@ def test_oardel_simple_cosystem(minimal_db_initialization, setup_config): job_id = insert_job( minimal_db_initialization, res=[(60, [("resource_id=4", "")])], - types=["cosystem"], + types=["cosystem", "lol"], state="Running", ) runner = CliRunner() result = runner.invoke( - cli, ["-s", "USR1", str(job_id)], obj=(minimal_db_initialization, config) + cli, + ["-s", "USR1", str(job_id)], + obj=(minimal_db_initialization, config), + catch_exceptions=True, ) print(result.output) import traceback diff --git a/tests/cli/test_oarstat.py b/tests/cli/test_oarstat.py index fa71880b..bf3ae017 100644 --- a/tests/cli/test_oarstat.py +++ b/tests/cli/test_oarstat.py @@ -563,3 +563,28 @@ def test_oarstat_job_id_error(minimal_db_initialization, setup_config): except ValueError: assert False assert result.exit_code == 0 + + +def test_oarstat_job_types(minimal_db_initialization, setup_config): + jid = insert_job( + minimal_db_initialization, + res=[(60, [("resource_id=4", "")])], + user="toto", + types=["inner=4", "cosystem"], + properties="", + ) + + runner = CliRunner() + result = runner.invoke( + cli, + ["-j", f"{jid}", "-f"], + obj=minimal_db_initialization, + catch_exceptions=False, + ) + print(result.output) + for line in result.output.splitlines(): + if line.startswith(" types = "): + line = line.strip() + assert line == "types = inner=4, cosystem" + + assert result.exit_code == 0