info-dynamic-condor

#!/usr/bin/env python
# coding: utf8

import htcondor
import argparse
import datetime


class InfoDynamicCondor(object):
    """
    InfoDynamicCondor is a object to store usefull data
    """
    def __init__(self, scheduler_address):
        """
        Initialize InfoDynamicCondor
        """
        self.name = "condor"
        self.scheduler_address = scheduler_address
        self.version = htcondor.version()[16:22]
        self.cpus = 0
        self.free = 0
        self.vo = {}
        # TODO: This value should be compute
        self.max_waiting = 2000
        self.max_memory = 2048
        self.max_slot_per_job = 8
        self.max_virtual_memory = 3072
        self.max_wall_clock_time = 2160
        self.max_cpu_time = 2160
        self.status = 'Production'

    def load(self):
        """
        Retrieve information from HTCondor and populate object with it
        """
        collector = htcondor.Collector()

        # Collector provide information about worker node state
        worker_nodes = collector.query(htcondor.AdTypes.Startd)
        self.total_cpus(worker_nodes)
        self.total_free(worker_nodes)

        # Scheduler provide information about jobs
        scheduler_address = collector.locate(htcondor.DaemonTypes.Schedd, self.scheduler_address)
        scheduler = htcondor.Schedd(scheduler_address)
        jobs = scheduler.query()
        self.per_vo_jobs(jobs)

    def total_cpus(self, worker_nodes):
        """
        cpus is just a addition of Cpus find on all Worker Node
        """
        for worker_node in worker_nodes:
            self.cpus += worker_node['Cpus']

    def total_free(self, worker_nodes):
        """
        Each Cpus in worker node unclaimed is considered free
        """
        for worker_node in worker_nodes:
            if worker_node['State'] == "Unclaimed":
                self.free += worker_node['Cpus']

    def total_running(self):
        """
        Running job is a addition of all job running for each vo
        """
        running = 0
        for vo in self.vo:
            running += self.vo[vo]['running']
        return running

    def total_waiting(self):
        """
        Waiting job is a addition of all job waiting for each vo
        """
        waiting = 0
        for vo in self.vo:
            waiting += self.vo[vo]['waiting']
        return waiting

    def per_vo_jobs(self, jobs):
        """
        Compute information about job for each VO
        """
        for job in jobs:
            # We only considere Running and Idle job, we ignore any other state (completed, ...)
            if job['JobStatus'] == 1 or job['JobStatus'] == 2:
                # If this is the first job for this VO we create a empty entry
                if job['x509UserProxyVOName'] not in self.vo:
                    self.vo[job['x509UserProxyVOName']] = {'total': 0,
                                                           'running': 0,
                                                           'waiting': 0
                                                           }
                # TODO: Is it really useful ?
                self.vo[job['x509UserProxyVOName']]['total'] += 1
                if job['JobStatus'] == 1:
                    self.vo[job['x509UserProxyVOName']]['waiting'] += 1
                else:
                    self.vo[job['x509UserProxyVOName']]['running'] += 1


class ConsoleView(object):
    """
    Class to display a human reading format of status
    """
    @staticmethod
    def display(dynamic_information):
        information = dynamic_information.__dict__
        information['waiting'] = dynamic_information.total_waiting()
        information['running'] = dynamic_information.total_running()
        print "######################\n" \
              "# Global Information #\n" \
              "######################\n" \
              "Total CPUs:   {cpus}\n" \
              "Free CPUS:    {free}\n" \
              "Waiting Jobs: {waiting}\n" \
              "Running Jobs: {running}\n" \
              "".format(**information)
        for vo in information['vo']:
            vo_information = information['vo'][vo]
            vo_information['name'] = vo
            print "    * {name}\n" \
                  "    Waiting Jobs: {waiting}\n" \
                  "    Running Jobs: {running}\n" \
                  "".format(**vo_information)


class Glue1View(object):
    """
    Class to display into GLUE1 information
    """
    @staticmethod
    def display(dynamic_information, glue1_ldif):
        glue_file = open(glue1_ldif, "r")
        for line in glue_file:
            if line.startswith('dn: GlueCEUniqueID='):
                print line + "" \
                             "GlueCEInfoTotalCPUs: {cpus}\n" \
                             "GlueCEPolicyAssignedJobSlots: {cpus}\n" \
                             "GlueCEStateFreeCPUs: {free}\n" \
                             "GlueCEStateFreeJobSlots: {free}\n" \
                             "GlueCEPolicyMaxRunningJobs: {cpus}\n" \
                             "GlueCEPolicyMaxWallClockTime: {max_wall_clock_time}\n" \
                             "GlueCEPolicyMaxCPUTime: {max_cpu_time}\n" \
                             "GlueCEStateStatus: {status}\n" \
                             "GlueCEInfoLRMSVersion: {version}\n" \
                             "GlueCEInfoLRMSType: {name}".format(**dynamic_information.__dict__)
            if line.startswith('dn: GlueVOViewLocalID='):
                # TODO: Add a per-vo output (VOView)
                pass
        file.close()


class Glue2View(object):
    """
    Class to display into GLUE2 information
    """
    @staticmethod
    def display(dynamic_information, glue2_ldif):
        # TODO: Make a per-vo output instead of global output
        information = dynamic_information.__dict__
        information['waiting'] = dynamic_information.total_waiting()
        information['max_total'] = dynamic_information.max_waiting + dynamic_information.cpus
        information['current_date'] = datetime.datetime.now().isoformat()
        glue_file = open(glue2_ldif, "r")
        for line in glue_file:
            if line.startswith('dn: GLUE2ShareID'):
                print line + "" \
                             "GLUE2ComputingShareFreeSlots: {free}\n" \
                             "GLUE2ComputingShareMaxRunningJobs: {cpus}\n" \
                             "GLUE2ComputingShareMaxWaitingJobs: {max_waiting}\n" \
                             "GLUE2ComputingShareMaxTotalJobs: {max_total}\n" \
                             "GLUE2ComputingShareServingState: {status}\n" \
                             "GLUE2EntityCreationTime: {current_date}\n" \
                             "GLUE2ComputingShareMaxWallTime: {max_wall_clock_time}\n" \
                             "GLUE2ComputingShareDefaultWallTime: {max_wall_clock_time}\n" \
                             "GLUE2ComputingShareMaxCPUTime: {max_cpu_time}\n" \
                             "GLUE2ComputingShareDefaultCPUTime: {max_cpu_time}\n" \
                             "GLUE2ComputingShareMaxSlotsPerJob: {max_slot_per_job}\n" \
                             "GLUE2ComputingShareMaxMainMemory: {max_memory}\n" \
                             "GLUE2ComputingShareMaxVirtualMemory: {max_virtual_memory}" \
                             "".format(**information)
        file.close()


def main():
    """
    Main function for info-dynamic-condor
    """
    ###################
    # Argument parser #
    ###################
    parser = argparse.ArgumentParser(description="info-dynamic-condor return a GLUE1 / GLUE2 output"
                                                 "of condor scheduler state")
    parser.add_argument('--scheduler', dest='scheduler',
                        help='Address of scheduler')
    parser.add_argument('--glue1-ldif', dest='glue1_ldif',
                        help='Path to GLUE1 ldif file')
    parser.add_argument('--glue2-ldif', dest='glue2_ldif',
                        help='Path to GLUE2 ldif file')
    parser.add_argument('--console', dest='console', action='store_true',
                        help='Display Information w/ Human Reading output', default=False)
    options = parser.parse_args()

    ##############################################
    # Retrieve dynamic information from HTCondor #
    ##############################################
    dynamic_information = InfoDynamicCondor(options.scheduler)
    dynamic_information.load()

    #######################
    # Display information #
    #######################
    # print "ConsoleView"`
    if options.console:
        ConsoleView.display(dynamic_information)
    if options.glue1_ldif is not None:
        Glue1View.display(dynamic_information, options.glue1_ldif)
    if options.glue2_ldif is not None:
        Glue2View.display(dynamic_information, options.glue2_ldif)

if __name__ == '__main__':
    main()