Skip to content

Commit

Permalink
Added output_metadata() and tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@jdh4
jdh4 committed Nov 2, 2024
1 parent 988170a commit e4452a8
Show file tree
Hide file tree
Showing 2 changed files with 97 additions and 43 deletions.
85 changes: 50 additions & 35 deletions output_formatters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ def __init__(self, js: Jobstats) -> None:
def output(self, no_color: bool=True) -> str:
pass

@abstractmethod
def output_metadata(self) -> str:
pass

@staticmethod
def human_bytes(size: int, decimal_places=1) -> str:
size = float(size)
Expand Down Expand Up @@ -55,7 +59,7 @@ def human_seconds(seconds: int) -> str:
return "%s%02d:%02d" % (hour, minutes, seconds)

@staticmethod
def human_datetime(seconds_since_epoch):
def human_datetime(seconds_since_epoch: int) -> str:
fmt = "%a %b %-d, %Y at %-I:%M %p"
return datetime.datetime.fromtimestamp(seconds_since_epoch).strftime(fmt)

Expand Down Expand Up @@ -114,7 +118,7 @@ def rounded_memory_with_safety(mem_used: float) -> int:
return max(1, mem_with_safety)
return mem_suggested

def time_limit_formatted(self):
def time_limit_formatted(self) -> str:
self.js.time_eff_violation = False
clr = self.txt_normal
if self.js.state == "COMPLETED" and self.js.timelimitraw > 0:
Expand All @@ -128,7 +132,7 @@ def time_limit_formatted(self):
hs = self.human_seconds(SECONDS_PER_MINUTE * self.js.timelimitraw)
return f" Time Limit: {clr}{hs}{self.txt_normal}"

def draw_meter(self, efficiency, hardware, util=False):
def draw_meter(self, efficiency:int, hardware:str, util: bool=False) -> str:
bars = efficiency // 2
if bars < 0:
bars = 0
Expand Down Expand Up @@ -187,8 +191,9 @@ def format_note(self, *items, style="normal", indent_width=4, bullet="*") -> str
return f"{styling}{note}{self.txt_normal}{newlines}"

def job_notes(self):
"""Process the notes in config.py."""
s = ""
# compute several quantities which can then referenced in notes
# compute several quantities which can later be referenced in notes
total_used, total, total_cores = self.js.cpu_mem_total__used_alloc_cores
cores_per_node = int(self.js.ncpus) / int(self.js.nnodes)
gb_per_core_used = total_used / total_cores / 1024**3 if total_cores != 0 else 0
Expand All @@ -201,8 +206,10 @@ def job_notes(self):
zero_gpu = False # unused
zero_cpu = False # unused
gpu_show = True # unused
# low GPU utilization
interactive_job = "sys/dashboard/sys/" in self.js.jobname or self.js.jobname == "interactive"
# interactive job
cond1 = bool("sys/dashboard/sys/" in self.js.jobname)
cond2 = bool(self.js.jobname == "interactive")
interactive_job = bool(cond1 or cond2)
# low cpu utilization
somewhat = " " if self.js.cpu_efficiency < c.CPU_UTIL_RED else " somewhat "
ceff = self.js.cpu_efficiency if self.js.cpu_efficiency > 0 else "less than 1"
Expand All @@ -212,8 +219,10 @@ def job_notes(self):
approx = " approximately " if self.js.cpu_efficiency != round(eff_if_serial) else " "
# next four lines needed for excess CPU memory note
gb_per_core = total / total_cores / 1024**3 if total_cores != 0 else 0
opening = f"only used {self.js.cpu_memory_efficiency}%" if self.js.cpu_memory_efficiency >= 1 \
else "used less than 1%"
if self.js.cpu_memory_efficiency >= 1:
opening = f"only used {self.js.cpu_memory_efficiency}%"
else:
opening = "used less than 1%"
if self.js.cluster in c.CORES_PER_NODE:
cpn = c.CORES_PER_NODE[self.js.cluster]
else:
Expand Down Expand Up @@ -241,11 +250,33 @@ def job_notes(self):
class ClassicOutput(BaseFormatter):
"""Classic output formatter for the job report."""

def __init__(self, js: Jobstats):
def __init__(self, js: Jobstats, width: int=80) -> None:
super().__init__(js)
self.txt_bold = ""
self.txt_red = ""
self.txt_normal = ""
self.width = width

def output_metadata(self) -> str:
meta = f" Job ID: {self.txt_bold}{self.js.jobid}{self.txt_normal}\n"
meta += f" NetID/Account: {self.js.user}/{self.js.account}\n"
meta += f" Job Name: {self.js.jobname}\n"
if self.js.state in ("OUT_OF_MEMORY", "TIMEOUT"):
meta += f" State: {self.txt_bold}{self.txt_red}{self.js.state}{self.txt_normal}\n"
else:
meta += f" State: {self.js.state}\n"
meta += f" Nodes: {self.js.nnodes}\n"
meta += f" CPU Cores: {self.js.ncpus}\n"
meta += self.cpu_memory_formatted() + "\n"
if self.js.gpus:
meta += f" GPUs: {self.js.gpus}\n"
meta += f" QOS/Partition: {self.js.qos}/{self.js.partition}\n"
meta += f" Cluster: {self.js.cluster}\n"
meta += f" Start Time: {self.human_datetime(self.js.start)}\n"
in_progress = " (in progress)" if self.js.state == "RUNNING" else ""
meta += f" Run Time: {self.human_seconds(self.js.diff)}{in_progress}\n"
meta += self.time_limit_formatted() + "\n"
return meta

def output(self, no_color: bool=True) -> str:
if blessed_is_available and not no_color:
Expand All @@ -257,33 +288,17 @@ def output(self, no_color: bool=True) -> str:
# JOB METADATA #
########################################################################
report = "\n"
report += 80 * "=" + "\n"
report += " Slurm Job Statistics\n"
report += 80 * "=" + "\n"
report += f" Job ID: {self.txt_bold}{self.js.jobid}{self.txt_normal}\n"
report += f" NetID/Account: {self.js.user}/{self.js.account}\n"
report += f" Job Name: {self.js.jobname}\n"
if self.js.state in ("OUT_OF_MEMORY", "TIMEOUT"):
report += f" State: {self.txt_bold}{self.txt_red}{self.js.state}{self.txt_normal}\n"
else:
report += f" State: {self.js.state}\n"
report += f" Nodes: {self.js.nnodes}\n"
report += f" CPU Cores: {self.js.ncpus}\n"
report += self.cpu_memory_formatted() + "\n"
if self.js.gpus:
report += f" GPUs: {self.js.gpus}\n"
report += f" QOS/Partition: {self.js.qos}/{self.js.partition}\n"
report += f" Cluster: {self.js.cluster}\n"
report += f" Start Time: {self.human_datetime(self.js.start)}\n"
in_progress = " (in progress)" if self.js.state == "RUNNING" else ""
report += f" Run Time: {self.human_seconds(self.js.diff)}{in_progress}\n"
report += self.time_limit_formatted() + "\n"
report += self.width * "=" + "\n"
report += "Slurm Job Statistics".center(self.width) + "\n"
report += self.width * "=" + "\n"
report += self.output_metadata()
report += "\n"
report += f" {self.txt_bold}Overall Utilization{self.txt_normal}\n"
report += 80 * "=" + "\n"

########################################################################
# OVERALL UTILIZATION #
########################################################################
report += f" {self.txt_bold}Overall Utilization{self.txt_normal}\n"
report += self.width * "=" + "\n"
# overall CPU time utilization
if self.js.cpu_util_error_code == 0:
total_used, total, _ = self.js.cpu_util_total__used_alloc_cores
Expand Down Expand Up @@ -343,7 +358,7 @@ def output(self, no_color: bool=True) -> str:
# DETAILED UTILIZATION #
########################################################################
report += f" {self.txt_bold}Detailed Utilization{self.txt_normal}\n"
report += 80 * "=" + "\n"
report += self.width * "=" + "\n"
gutter = " "
# CPU time utilization
report += f"{gutter}CPU utilization per node (CPU time used/run time)\n"
Expand Down Expand Up @@ -409,13 +424,13 @@ def output(self, no_color: bool=True) -> str:
notes = self.job_notes()
if notes:
report += f" {self.txt_bold}Notes{self.txt_normal}\n"
report += 80 * "=" + "\n"
report += self.width * "=" + "\n"
report += notes
return report
else:
report += "\n"
report += f" {self.txt_bold}Notes{self.txt_normal}\n"
report += 80 * "=" + "\n"
report += self.width * "=" + "\n"
if self.js.cpu_util_error_code:
report += f"{gutter}* The CPU utilization could not be determined.\n"
if self.js.cpu_mem_error_code:
Expand Down
55 changes: 47 additions & 8 deletions tests/test_output_formatters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,15 @@ def simple_stats(mocker):
ss64 = ('JS1:H4sIADelIWcC/1WNQQqDMBBF7zLrtEzG0ZhcphQzqGBM0bgQyd0bUii4fe8'
'//gVr9LKDuyDNo2yPibpBr00FMb2XV5AQtxOcRtMY1j0xKjh28X/TdsRMhRfxa9'
'IcBJyxbPsnKRg+R3lgzPk+ICSrYKwW8xcnjeJ8iwAAAA==')
data = ('10920562|1730212549|Unknown|tiger2|billing=40,cpu=40,mem=10G,no'
'de=1|%s|aturing|physics|RUNNING|1|40|10G|tiger-short|serial|144'
'0|myjob\n' % ss64)
ss64 = ('JS1:H4sIAPdcJmcC/1WNQQqDMBBF7zLrtEzG0ZhcphQzqGBM0bgQyd0bUii4fe8'
'//gVr9LKDuyDNo2yPibpBr00FMb2XV5AQtxOcRtMY1j0xKjh28X/TdsRMhRfxa9'
'IcBJyxbPsnKRg+R3lgzPk+ICSrYKwW8xcnjeJ8iwAAAA==')
data = ('10920562|1730212549|1730214578|tiger2|billing=40,cpu=40,mem=10G,no'
'de=1|%s|aturing|physics|COMPLETED|1|40|10G|tiger-short|serial|144'
'0|9\n' % ss64)
sacct_bytes = bytes(cols + data, "utf-8")
mocker.patch("subprocess.check_output", return_value=sacct_bytes)
stats = Jobstats(jobid="DUMMY-JOBID", prom_server="DUMMY-SERVER")
stats = Jobstats(jobid="10920562", prom_server="DUMMY-SERVER")
return stats


Expand Down Expand Up @@ -65,7 +68,7 @@ def test_cpu_memory_formatted(simple_stats):
assert formatter.cpu_memory_formatted(with_label=False) == "10"
formatter.js.reqmem = "10000G"
assert formatter.cpu_memory_formatted(with_label=False) == "10TB"
formatter.js.reqmem = "100.5G"
formatter.js.reqmem = "100.50G"
formatter.js.ncpus = 9
expected = " CPU Memory: 100.5GB (11.2GB per CPU-core)"
assert formatter.cpu_memory_formatted() == expected
Expand All @@ -90,6 +93,42 @@ def test_time_limit_formatted(simple_stats):

def test_draw_meter(simple_stats):
formatter = ClassicOutput(simple_stats)
bars = "|" * (75 // 2)
spaces = " " * (50 - len(bars) - 3)
assert formatter.draw_meter(75, "cpu") == f"[{bars}{spaces}75%]"
expected = "[ 0%]"
assert formatter.draw_meter(0, "cpu") == expected
expected = "[||||||||||||||||||||||||||||||||||||| 75%]"
assert formatter.draw_meter(75, "cpu") == expected
expected = "[||||||||||||||||||||||||||||||||||||||||||||||100%]"
assert formatter.draw_meter(100, "cpu") == expected


def test_format_note(simple_stats):
formatter = ClassicOutput(simple_stats)
note = "A simple note."
expected = f" * {note}\n\n"
assert formatter.format_note(note) == expected
note = "A simple note:"
url = "https://mysite.ext"
expected = f" * {note}\n {url}\n\n"
assert formatter.format_note(note, url) == expected

def test_output_metadata(simple_stats):
formatter = ClassicOutput(simple_stats)
expected = """
Job ID: 10920562
NetID/Account: aturing/physics
Job Name: 9
State: COMPLETED
Nodes: 1
CPU Cores: 40
CPU Memory: 10GB (250MB per CPU-core)
QOS/Partition: tiger-short/serial
Cluster: tiger
Start Time: Tue Oct 29, 2024 at 10:35 AM
Run Time: 00:33:49
Time Limit: 1-00:00:00
"""
actual = formatter.output_metadata()
for e, a in zip(expected.split("\n"), [""] + actual.split("\n")):
# avoid timezone complications
if "Start Time" not in e:
assert e.strip() == a.strip()

0 comments on commit e4452a8

Please sign in to comment.