WorkflowPercentage.py

#!/usr/bin/env python
import json
import urllib2,urllib, httplib, sys, re, os
from xml.dom.minidom import getDOMImplementation
import reqMgrClient, dbs3Client
from optparse import OptionParser
"""
    Calculates event progress percentage of a given workflow,
    taking into account the workflow type, and comparing
    input vs. output numbers of events.
    Should be used instead of dbsTest.py
    TODO MC??
   
"""

def percentageCompletion(url, workflow, verbose=False, checkLumis=False, checkFilter=False):
    """
    Calculates Percentage of completion for a given workflow
    taking a particular output dataset
    if checkLumis is enabled, we get lumis instead.
    """
    #input events/lumis
    try:
        if checkLumis:
            inputEvents = workflow.getInputLumis()
        else:
            inputEvents = workflow.getInputEvents()
    except:
        #no input dataset
        inputEvents = 0        
    
    #filter Efficiency (only for events)
    if checkFilter and not checkLumis and 'FilterEfficiency' in workflow.info:
        filterEff = workflow.filterEfficiency
    else:
        filterEff = 1.0
    #datasets
    for dataset in workflow.outputDatasets:
        #output events/lumis
        if checkLumis:
            outputEvents = workflow.getOutputLumis(dataset)
        else:
            outputEvents = workflow.getOutputEvents(dataset)
        if not outputEvents:
            outputEvents = 0
        #calculate percentage
        if not inputEvents:
            perc = 0
        else:
            perc = 100.0*outputEvents/float(inputEvents)/filterEff
        #print results
        if verbose:
            print dataset
            print "Input %s: %d"%("lumis" if checkLumis else "events", int(inputEvents))
            print ("Output %s: %d (%s%%)"%("lumis" if checkLumis else "events", int(outputEvents), perc)+
                   ('(filter=%s)'%filterEff if checkFilter and not checkLumis else ''))
        else:
            print dataset, "%s%%"%perc

def percentageCompletion2StepMC(url, workflow, verbose=False, checkLumis=False):
    """
    Calculates percentage completion for a MonteCarlo
    with GEN and GEN-SIM output
    pdmvserv_SMP-Summer14Test2wmGENSIM-00002_00002_v0__140831_173202_4712
    """
    #input events/lumis
    try:
        if checkLumis:
            inputEvents = workflow.getInputLumis()
        else:
            inputEvents = workflow.getInputEvents()
    except:
        #no input dataset
        inputEvents = 0
    
    #filter Efficiency (only for events)
    if not checkLumis and 'FilterEfficiency' in workflow.info:
        filterEff = workflow.filterEfficiency
    else:
        filterEff = 1.0

    #set the GEN first
    if re.match('.*/GEN$', workflow.outputDatasets[1]):
        workflow.outputDatasets = [workflow.outputDatasets[1],workflow.outputDatasets[0]]
    #output events/lumis
    if checkLumis:
        outputEvents = [ workflow.getOutputEvents(workflow.outputDatasets[0]),
                        workflow.getOutputEvents(workflow.outputDatasets[1])]
    else:
        outputEvents = [ workflow.getOutputLumis(workflow.outputDatasets[0]),
                        workflow.getOutputLumis(workflow.outputDatasets[1])]
    if not inputEvents:
        perc = [100.0,100.0*outputEvents[1]/outputEvents[0]]
    else:
        perc = [100.0*outputEvents[0]/float(inputEvents),
                100.0*outputEvents[1]/float(inputEvents)/filterEff]
    #print results
    if verbose:
        print "Input %s: %d"%("lumis" if checkLumis else "events", int(inputEvents))
        print workflow.outputDatasets[0]
        print "Output %s: %d (%s%%)"%("lumis" if checkLumis else "events", int(outputEvents[0]), perc[0])
        print workflow.outputDatasets[1]
        print ("Output %s: %d (%s%%)"%("lumis" if checkLumis else "events", int(outputEvents[1]), perc[1])+
              ('(filter=%s)'%filterEff if not checkLumis else ''))
    else:
        print workflow.outputDatasets[0], "%s%%"%perc[0]
        print workflow.outputDatasets[1], "%s%%"%perc[1]

def percentageCompletionTaskChain(url, workflow, verbose=False, checkLumis=False):
    """
    Calculates a Percentage completion for a taskchain.
    Taking step/filter efficiency into account.
    pdmvserv_task_SUS-Summer12WMLHE-00004__v1_T_141003_120119_9755
    """
    workflow = reqMgrClient.TaskChain(workflow.name)
    if checkLumis:
        inputEvents = workflow.getInputLumis()
    else:
        inputEvents = workflow.getInputEvents()
    if verbose:
        print "Input %s:"%("lumis" if checkLumis else "events"), inputEvents
    i = 1

    #task-chain 1, starts with GEN, a GEN-SIM, GEN-SIM-RAW, AODSIM, DQM and so on
    for dataset in workflow.outputDatasets:
        if verbose:
            print dataset
        if not checkLumis:
            outputEvents = workflow.getOutputEvents(dataset)
        else:
            outputEvents = workflow.getOutputLumis(dataset)
        #GEN and GEN-SIM and events, we take into account filter efficiency
        if 1<= i <= 2 and not checkLumis: 
            filterEff = workflow.getFilterEfficiency('Task%d'%i)
            #decrease filter eff
            inputEvents *= filterEff
            percentage = 100.0*outputEvents/float(inputEvents) if inputEvents > 0 else 0.0
            if verbose:
                print "Output %s:"%("lumis" if checkLumis else "events"), int(outputEvents), "(%.2f%%)"%percentage
        #Other datasets, or lumis, we ignore filter efficiency
        else:
            percentage = 100.0*outputEvents/float(inputEvents) if inputEvents > 0 else 0.0
            if verbose:
                print "Output %s:"%("lumis" if checkLumis else "events"), int(outputEvents), "(%.2f%%)"%percentage
        if not verbose:
            print dataset, "%s%%"%percentage
        i += 1

url = 'cmsweb.cern.ch'

def main():
    usage = "usage: %prog [options] workflow"
    parser = OptionParser(usage=usage)
    parser.add_option("-v","--verbose",action="store_true", dest="verbose", default=False,
                        help="Show detailed info")
    parser.add_option("-l","--lumis",action="store_true", dest="checkLumis", default=False,
                        help="Show lumis instead of events")
    parser.add_option("-f","--file", dest="fileName", default=None,
                        help="Input file")
    (options, args) = parser.parse_args()
    if len(args) != 1 and options.fileName is None:
        parser.error("Provide the workflow name or a file")
        sys.exit(1)
    if options.fileName is None:
        workflows = [args[0]]
    else:
        workflows = [l.strip() for l in open(options.fileName) if l.strip()]

    for wf in workflows:
        print wf
        workflow = reqMgrClient.Workflow(wf, url)        
        #by tyoe
        if workflow.type != 'TaskChain':
            #two step monte carlos (GEN and GEN-SIM)
            if workflow.type == 'MonteCarlo' and len(workflow.outputDatasets) == 2:
                percentageCompletion2StepMC(url, workflow, options.verbose, options.checkLumis)
            elif workflow.type == 'MonteCarloFromGEN':
                percentageCompletion(url, workflow, options.verbose, options.checkLumis, checkFilter=True)
            else:
                percentageCompletion(url, workflow, options.verbose, options.checkLumis, checkFilter=True)
        else:
            percentageCompletionTaskChain(url, workflow, options.verbose, options.checkLumis)

if __name__ == "__main__":
    main()