diff --git a/info-dynamic-condor b/info-dynamic-condor new file mode 100755 index 0000000..c7c8fd9 --- /dev/null +++ b/info-dynamic-condor @@ -0,0 +1,224 @@ +#!/usr/bin/env python +# coding: utf8 + +import htcondor +import argparse +import datetime + + +class InfoDynamicCondor(object): + """ + InfoDynamicCondor is a object to store usefull data + """ + def __init__(self, scheduler_address): + """ + Initialize InfoDynamicCondor + """ + self.name = "condor" + self.scheduler_address = scheduler_address + self.version = htcondor.version()[16:22] + self.cpus = 0 + self.free = 0 + self.vo = {} + # TODO: This value should be compute + self.max_waiting = 2000 + self.max_memory = 2048 + self.max_slot_per_job = 8 + self.max_virtual_memory = 3072 + self.max_wall_clock_time = 2160 + self.max_cpu_time = 2160 + self.status = 'Production' + + def load(self): + """ + Retrieve information from HTCondor and populate object with it + """ + collector = htcondor.Collector() + + # Collector provide information about worker node state + worker_nodes = collector.query(htcondor.AdTypes.Startd) + self.total_cpus(worker_nodes) + self.total_free(worker_nodes) + + # Scheduler provide information about jobs + scheduler_address = collector.locate(htcondor.DaemonTypes.Schedd, self.scheduler_address) + scheduler = htcondor.Schedd(scheduler_address) + jobs = scheduler.query() + self.per_vo_jobs(jobs) + + def total_cpus(self, worker_nodes): + """ + cpus is just a addition of Cpus find on all Worker Node + """ + for worker_node in worker_nodes: + self.cpus += worker_node['Cpus'] + + def total_free(self, worker_nodes): + """ + Each Cpus in worker node unclaimed is considered free + """ + for worker_node in worker_nodes: + if worker_node['State'] == "Unclaimed": + self.free += worker_node['Cpus'] + + def total_running(self): + """ + Running job is a addition of all job running for each vo + """ + running = 0 + for vo in self.vo: + running += self.vo[vo]['running'] + return running + + def total_waiting(self): + """ + Waiting job is a addition of all job waiting for each vo + """ + waiting = 0 + for vo in self.vo: + waiting += self.vo[vo]['waiting'] + return waiting + + def per_vo_jobs(self, jobs): + """ + Compute information about job for each VO + """ + for job in jobs: + # We only considere Running and Idle job, we ignore any other state (completed, ...) + if job['JobStatus'] == 1 or job['JobStatus'] == 2: + # If this is the first job for this VO we create a empty entry + if job['x509UserProxyVOName'] not in self.vo: + self.vo[job['x509UserProxyVOName']] = {'total': 0, + 'running': 0, + 'waiting': 0 + } + # TODO: Is it really useful ? + self.vo[job['x509UserProxyVOName']]['total'] += 1 + if job['JobStatus'] == 1: + self.vo[job['x509UserProxyVOName']]['waiting'] += 1 + else: + self.vo[job['x509UserProxyVOName']]['running'] += 1 + + +class ConsoleView(object): + """ + Class to display a human reading format of status + """ + @staticmethod + def display(dynamic_information): + information = dynamic_information.__dict__ + information['waiting'] = dynamic_information.total_waiting() + information['running'] = dynamic_information.total_running() + print "######################\n" \ + "# Global Information #\n" \ + "######################\n" \ + "Total CPUs: {cpus}\n" \ + "Free CPUS: {free}\n" \ + "Waiting Jobs: {waiting}\n" \ + "Running Jobs: {running}\n" \ + "".format(**information) + for vo in information['vo']: + vo_information = information['vo'][vo] + vo_information['name'] = vo + print " * {name}\n" \ + " Waiting Jobs: {waiting}\n" \ + " Running Jobs: {running}\n" \ + "".format(**vo_information) + + +class Glue1View(object): + """ + Class to display into GLUE1 information + """ + @staticmethod + def display(dynamic_information, glue1_ldif): + glue_file = open(glue1_ldif, "r") + for line in glue_file: + if line.startswith('dn: GlueCEUniqueID='): + print line + "" \ + "GlueCEInfoTotalCPUs: {cpus}\n" \ + "GlueCEPolicyAssignedJobSlots: {cpus}\n" \ + "GlueCEStateFreeCPUs: {free}\n" \ + "GlueCEStateFreeJobSlots: {free}\n" \ + "GlueCEPolicyMaxRunningJobs: {cpus}\n" \ + "GlueCEPolicyMaxWallClockTime: {max_wall_clock_time}\n" \ + "GlueCEPolicyMaxCPUTime: {max_cpu_time}\n" \ + "GlueCEStateStatus: {status}\n" \ + "GlueCEInfoLRMSVersion: {version}\n" \ + "GlueCEInfoLRMSType: {name}".format(**dynamic_information.__dict__) + if line.startswith('dn: GlueVOViewLocalID='): + # TODO: Add a per-vo output (VOView) + pass + file.close() + + +class Glue2View(object): + """ + Class to display into GLUE2 information + """ + @staticmethod + def display(dynamic_information, glue2_ldif): + # TODO: Make a per-vo output instead of global output + information = dynamic_information.__dict__ + information['waiting'] = dynamic_information.total_waiting() + information['max_total'] = dynamic_information.max_waiting + dynamic_information.cpus + information['current_date'] = datetime.datetime.now().isoformat() + glue_file = open(glue2_ldif, "r") + for line in glue_file: + if line.startswith('dn: GLUE2ShareID'): + print line + "" \ + "GLUE2ComputingShareFreeSlots: {free}\n" \ + "GLUE2ComputingShareMaxRunningJobs: {cpus}\n" \ + "GLUE2ComputingShareMaxWaitingJobs: {max_waiting}\n" \ + "GLUE2ComputingShareMaxTotalJobs: {max_total}\n" \ + "GLUE2ComputingShareServingState: {status}\n" \ + "GLUE2EntityCreationTime: {current_date}\n" \ + "GLUE2ComputingShareMaxWallTime: {max_wall_clock_time}\n" \ + "GLUE2ComputingShareDefaultWallTime: {max_wall_clock_time}\n" \ + "GLUE2ComputingShareMaxCPUTime: {max_cpu_time}\n" \ + "GLUE2ComputingShareDefaultCPUTime: {max_cpu_time}\n" \ + "GLUE2ComputingShareMaxSlotsPerJob: {max_slot_per_job}\n" \ + "GLUE2ComputingShareMaxMainMemory: {max_memory}\n" \ + "GLUE2ComputingShareMaxVirtualMemory: {max_virtual_memory}" \ + "".format(**information) + file.close() + + +def main(): + """ + Main function for info-dynamic-condor + """ + ################### + # Argument parser # + ################### + parser = argparse.ArgumentParser(description="info-dynamic-condor return a GLUE1 / GLUE2 output" + "of condor scheduler state") + parser.add_argument('--scheduler', dest='scheduler', + help='Address of scheduler') + parser.add_argument('--glue1-ldif', dest='glue1_ldif', + help='Path to GLUE1 ldif file') + parser.add_argument('--glue2-ldif', dest='glue2_ldif', + help='Path to GLUE2 ldif file') + parser.add_argument('--console', dest='console', action='store_true', + help='Display Information w/ Human Reading output', default=False) + options = parser.parse_args() + + ############################################## + # Retrieve dynamic information from HTCondor # + ############################################## + dynamic_information = InfoDynamicCondor(options.scheduler) + dynamic_information.load() + + ####################### + # Display information # + ####################### + # print "ConsoleView"` + if options.console: + ConsoleView.display(dynamic_information) + if options.glue1_ldif is not None: + Glue1View.display(dynamic_information, options.glue1_ldif) + if options.glue2_ldif is not None: + Glue2View.display(dynamic_information, options.glue2_ldif) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/setup.py b/setup.py index 870c7cf..680fb20 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ description='Set of python script to interact with HTCondor', author='Guillaume Philippon', author_email='guillaume.philippon@lal.in2p3.fr', - scripts=["csub"], + scripts=["csub", "info-dynamic-condor"], dependency_links=['http://research.cs.wisc.edu/htcondor/downloads/' '?state=select_from_mirror_page&version=8.2.7&' 'mirror=UW%20Madison&optional_organization_url=http://'],