Merge pull request #12 from guillaume-philippon/info-dynamic-condor

Create a info-dynamic-condor component to display Glue1 / Glue2 output

info-dynamic-condor

@@ -0,0 +1,224 @@
+#!/usr/bin/env python
+# coding: utf8
+
+import htcondor
+import argparse
+import datetime
+
+
+class InfoDynamicCondor(object):
+    """
+    InfoDynamicCondor is a object to store usefull data
+    """
+    def __init__(self, scheduler_address):
+        """
+        Initialize InfoDynamicCondor
+        """
+        self.name = "condor"
+        self.scheduler_address = scheduler_address
+        self.version = htcondor.version()[16:22]
+        self.cpus = 0
+        self.free = 0
+        self.vo = {}
+        # TODO: This value should be compute
+        self.max_waiting = 2000
+        self.max_memory = 2048
+        self.max_slot_per_job = 8
+        self.max_virtual_memory = 3072
+        self.max_wall_clock_time = 2160
+        self.max_cpu_time = 2160
+        self.status = 'Production'
+
+    def load(self):
+        """
+        Retrieve information from HTCondor and populate object with it
+        """
+        collector = htcondor.Collector()
+
+        # Collector provide information about worker node state
+        worker_nodes = collector.query(htcondor.AdTypes.Startd)
+        self.total_cpus(worker_nodes)
+        self.total_free(worker_nodes)
+
+        # Scheduler provide information about jobs
+        scheduler_address = collector.locate(htcondor.DaemonTypes.Schedd, self.scheduler_address)
+        scheduler = htcondor.Schedd(scheduler_address)
+        jobs = scheduler.query()
+        self.per_vo_jobs(jobs)
+
+    def total_cpus(self, worker_nodes):
+        """
+        cpus is just a addition of Cpus find on all Worker Node
+        """
+        for worker_node in worker_nodes:
+            self.cpus += worker_node['Cpus']
+
+    def total_free(self, worker_nodes):
+        """
+        Each Cpus in worker node unclaimed is considered free
+        """
+        for worker_node in worker_nodes:
+            if worker_node['State'] == "Unclaimed":
+                self.free += worker_node['Cpus']
+
+    def total_running(self):
+        """
+        Running job is a addition of all job running for each vo
+        """
+        running = 0
+        for vo in self.vo:
+            running += self.vo[vo]['running']
+        return running
+
+    def total_waiting(self):
+        """
+        Waiting job is a addition of all job waiting for each vo
+        """
+        waiting = 0
+        for vo in self.vo:
+            waiting += self.vo[vo]['waiting']
+        return waiting
+
+    def per_vo_jobs(self, jobs):
+        """
+        Compute information about job for each VO
+        """
+        for job in jobs:
+            # We only considere Running and Idle job, we ignore any other state (completed, ...)
+            if job['JobStatus'] == 1 or job['JobStatus'] == 2:
+                # If this is the first job for this VO we create a empty entry
+                if job['x509UserProxyVOName'] not in self.vo:
+                    self.vo[job['x509UserProxyVOName']] = {'total': 0,
+                                                           'running': 0,
+                                                           'waiting': 0
+                                                           }
+                # TODO: Is it really useful ?
+                self.vo[job['x509UserProxyVOName']]['total'] += 1
+                if job['JobStatus'] == 1:
+                    self.vo[job['x509UserProxyVOName']]['waiting'] += 1
+                else:
+                    self.vo[job['x509UserProxyVOName']]['running'] += 1
+
+
+class ConsoleView(object):
+    """
+    Class to display a human reading format of status
+    """
+    @staticmethod
+    def display(dynamic_information):
+        information = dynamic_information.__dict__
+        information['waiting'] = dynamic_information.total_waiting()
+        information['running'] = dynamic_information.total_running()
+        print "######################\n" \
+              "# Global Information #\n" \
+              "######################\n" \
+              "Total CPUs:   {cpus}\n" \
+              "Free CPUS:    {free}\n" \
+              "Waiting Jobs: {waiting}\n" \
+              "Running Jobs: {running}\n" \
+              "".format(**information)
+        for vo in information['vo']:
+            vo_information = information['vo'][vo]
+            vo_information['name'] = vo
+            print "    * {name}\n" \
+                  "    Waiting Jobs: {waiting}\n" \
+                  "    Running Jobs: {running}\n" \
+                  "".format(**vo_information)
+
+
+class Glue1View(object):
+    """
+    Class to display into GLUE1 information
+    """
+    @staticmethod
+    def display(dynamic_information, glue1_ldif):
+        glue_file = open(glue1_ldif, "r")
+        for line in glue_file:
+            if line.startswith('dn: GlueCEUniqueID='):
+                print line + "" \
+                             "GlueCEInfoTotalCPUs: {cpus}\n" \
+                             "GlueCEPolicyAssignedJobSlots: {cpus}\n" \
+                             "GlueCEStateFreeCPUs: {free}\n" \
+                             "GlueCEStateFreeJobSlots: {free}\n" \
+                             "GlueCEPolicyMaxRunningJobs: {cpus}\n" \
+                             "GlueCEPolicyMaxWallClockTime: {max_wall_clock_time}\n" \
+                             "GlueCEPolicyMaxCPUTime: {max_cpu_time}\n" \
+                             "GlueCEStateStatus: {status}\n" \
+                             "GlueCEInfoLRMSVersion: {version}\n" \
+                             "GlueCEInfoLRMSType: {name}".format(**dynamic_information.__dict__)
+            if line.startswith('dn: GlueVOViewLocalID='):
+                # TODO: Add a per-vo output (VOView)
+                pass
+        file.close()
+
+
+class Glue2View(object):
+    """
+    Class to display into GLUE2 information
+    """
+    @staticmethod
+    def display(dynamic_information, glue2_ldif):
+        # TODO: Make a per-vo output instead of global output
+        information = dynamic_information.__dict__
+        information['waiting'] = dynamic_information.total_waiting()
+        information['max_total'] = dynamic_information.max_waiting + dynamic_information.cpus
+        information['current_date'] = datetime.datetime.now().isoformat()
+        glue_file = open(glue2_ldif, "r")
+        for line in glue_file:
+            if line.startswith('dn: GLUE2ShareID'):
+                print line + "" \
+                             "GLUE2ComputingShareFreeSlots: {free}\n" \
+                             "GLUE2ComputingShareMaxRunningJobs: {cpus}\n" \
+                             "GLUE2ComputingShareMaxWaitingJobs: {max_waiting}\n" \
+                             "GLUE2ComputingShareMaxTotalJobs: {max_total}\n" \
+                             "GLUE2ComputingShareServingState: {status}\n" \
+                             "GLUE2EntityCreationTime: {current_date}\n" \
+                             "GLUE2ComputingShareMaxWallTime: {max_wall_clock_time}\n" \
+                             "GLUE2ComputingShareDefaultWallTime: {max_wall_clock_time}\n" \
+                             "GLUE2ComputingShareMaxCPUTime: {max_cpu_time}\n" \
+                             "GLUE2ComputingShareDefaultCPUTime: {max_cpu_time}\n" \
+                             "GLUE2ComputingShareMaxSlotsPerJob: {max_slot_per_job}\n" \
+                             "GLUE2ComputingShareMaxMainMemory: {max_memory}\n" \
+                             "GLUE2ComputingShareMaxVirtualMemory: {max_virtual_memory}" \
+                             "".format(**information)
+        file.close()
+
+
+def main():
+    """
+    Main function for info-dynamic-condor
+    """
+    ###################
+    # Argument parser #
+    ###################
+    parser = argparse.ArgumentParser(description="info-dynamic-condor return a GLUE1 / GLUE2 output"
+                                                 "of condor scheduler state")
+    parser.add_argument('--scheduler', dest='scheduler',
+                        help='Address of scheduler')
+    parser.add_argument('--glue1-ldif', dest='glue1_ldif',
+                        help='Path to GLUE1 ldif file')
+    parser.add_argument('--glue2-ldif', dest='glue2_ldif',
+                        help='Path to GLUE2 ldif file')
+    parser.add_argument('--console', dest='console', action='store_true',
+                        help='Display Information w/ Human Reading output', default=False)
+    options = parser.parse_args()
+
+    ##############################################
+    # Retrieve dynamic information from HTCondor #
+    ##############################################
+    dynamic_information = InfoDynamicCondor(options.scheduler)
+    dynamic_information.load()
+
+    #######################
+    # Display information #
+    #######################
+    # print "ConsoleView"`
+    if options.console:
+        ConsoleView.display(dynamic_information)
+    if options.glue1_ldif is not None:
+        Glue1View.display(dynamic_information, options.glue1_ldif)
+    if options.glue2_ldif is not None:
+        Glue2View.display(dynamic_information, options.glue2_ldif)
+
+if __name__ == '__main__':
+    main()

setup.py

@@ -8,7 +8,7 @@
       description='Set of python script to interact with HTCondor',
       author='Guillaume Philippon',
       author_email='[email protected]',
-      scripts=["csub"],
+      scripts=["csub", "info-dynamic-condor"],
       dependency_links=['http://research.cs.wisc.edu/htcondor/downloads/'
                         '?state=select_from_mirror_page&version=8.2.7&'
                         'mirror=UW%20Madison&optional_organization_url=http://'],