Integrate Pylint CI Checks and Migrate Codebase to Python 3

.github/workflows/pylint.yml

@@ -0,0 +1,23 @@
+name: Pylint
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pylint prettytable sqlalchemy
+    - name: Analysing the code with pylint
+      run: |
+        pylint --disable=all --enable=E $(git ls-files '*.py')

Tests/runTheGlobalTagTests.py

@@ -5,14 +5,14 @@
 import copy
 import string, re
 import subprocess
-import ConfigParser, json
+import configparser, json
 from optparse import OptionParser
 
 from autoCond_TEMPL import autoCondTemplate as myAutoCond
 
 ####################--- Classes ---############################
 
-class BetterConfigParser(ConfigParser.ConfigParser):
+class BetterConfigParser(configparser.ConfigParser):
 
     ##############################################
     def optionxform(self, optionstr):
@@ -38,7 +38,7 @@ def __updateDict( self, dictionary, section ):
             if "local"+section.title() in self.sections():
                 for option in self.options( "local"+section.title() ):
                     result[option] = self.get( "local"+section.title(),option )
-        except ConfigParser.NoSectionError, section:
+        except (ConfigParser.NoSectionError, section):
             msg = ("%s in configuration files. This section is mandatory."
                    %(str(section).replace(":", "", 1)))
             #raise AllInOneError(msg)
@@ -50,7 +50,7 @@ def getResultingSection( self, section, defaultDict = {}, demandPars = [] ):
         for option in demandPars:
             try:
                 result[option] = self.get( section, option )
-            except ConfigParser.NoOptionError, globalSectionError:
+            except (ConfigParser.NoOptionError, globalSectionError):
                 globalSection = str( globalSectionError ).split( "'" )[-2]
                 splittedSectionName = section.split( ":" )
                 if len( splittedSectionName ) > 1:
@@ -61,7 +61,7 @@ def getResultingSection( self, section, defaultDict = {}, demandPars = [] ):
                 if self.has_section( localSection ):
                     try:
                         result[option] = self.get( localSection, option )
-                    except ConfigParser.NoOptionError, option:
+                    except (ConfigParser.NoOptionError, option):
                         msg = ("%s. This option is mandatory."
                                %(str(option).replace(":", "", 1).replace(
                                    "section",
@@ -72,7 +72,7 @@ def getResultingSection( self, section, defaultDict = {}, demandPars = [] ):
                            %(str(globalSectionError).replace(":", "", 1)))
                     #raise AllInOneError(msg)
         result = self.__updateDict( result, section )
-        #print result
+        #print(result)
         return result
 
 ##### method to parse the input file ################################
@@ -94,7 +94,7 @@ def ConfigSectionMap(config, section):
 def replaceByMap(target, map):
     result = target
     for id in map:
-        #print "  "+id+": "+map[id]
+        #print("  "+id+": "+map[id])
         lifeSaver = 10e3
         iteration = 0
         while ".oO[" in result and "]Oo." in result:
@@ -103,23 +103,23 @@ def replaceByMap(target, map):
                 iteration += 1
             if iteration > lifeSaver:
                 problematicLines = ""
-                print map.keys()
+                print(map.keys())
                 for line in result.splitlines():
                     if  ".oO[" in result and "]Oo." in line:
                         problematicLines += "%s\n"%line
-                raise StandardError, "Oh Dear, there seems to be an endless loop in replaceByMap!!\n%s\nrepMap"%problematicLines
+                raise (StandardError, "Oh Dear, there seems to be an endless loop in replaceByMap!!\n%s\nrepMap"%problematicLines)
     return result
 
 ##############################################
 def execme(command,dryrun=False):    
     '''Wrapper for executing commands.
     '''
     if dryrun:
-        print command
+        print(command)
     else:
-        print " * Executing: %s..."%command
+        print(" * Executing: %s..."%command)
         os.system(command)
-        print " * Executed!"
+        print(" * Executed!")
 
 #####################################################################
 def getCommandOutput(command):
@@ -131,7 +131,7 @@ def getCommandOutput(command):
     data = child.read()
     err = child.close()
     if err:
-        print '%s failed w/ exit code %d' % (command, err)
+        print('%s failed w/ exit code %d' % (command, err))
     return data
 
 ##############################################
@@ -149,9 +149,9 @@ def main():
     repMap = {}
 
     if ConfigFile is not None:
-        print "********************************************************"
-        print "* Parsing from input file:", ConfigFile," "
-        print "********************************************************"
+        print("********************************************************")
+        print("* Parsing from input file:", ConfigFile," ")
+        print("********************************************************")
 
         config = BetterConfigParser()
         config.read(ConfigFile)
@@ -209,7 +209,7 @@ def main():
 
         for section in sections:
 
-            print "Preparing:",section
+            print("Preparing:",section)
             conditions = config.getResultingSection(section)
             if(conditions):
                 fout1.write("## "+section.replace("_"," ")+"\n \n")
@@ -223,25 +223,25 @@ def main():
                         fout2.write("   * =\'"+dict[key][0]+"\'= ("+dict[key][1]+") : [[https://cms-conddb.cern.ch/cmsDbBrowser/list/Prod/gts/"+params[0]+"]["+params[0]+"]],[[https://cms-conddb.cern.ch/cmsDbBrowser/diff/Prod/gts/"+params[0]+"/"+params[1]+"][diff with previous]]: \n")
                         fout2.write("      *\n \n")
 
-                        print "=====>",str(dict[key][0]).ljust(20),":",str(params[0]).ljust(20)
-                        #print '{:10s} {:10s}'.format(str(dict[key][0]),str(params[0]))
+                        print("=====>",str(dict[key][0]).ljust(20),":",str(params[0]).ljust(20))
+                        #print('{:10s} {:10s}'.format(str(dict[key][0]),str(params[0])))
                         repMap.update({dict[key][0].upper():params[0]})
 
         ## replace the map of inputs
 
         theReplacedMap = replaceByMap(myAutoCond,repMap)
-        #print repMap
-        #print theReplacedMap
+        #print(repMap)
+        #print(theReplacedMap)
 
         filePath = os.path.join(".","autoCond.py")
         theFile = open( filePath, "w" )
         theFile.write(theReplacedMap)
         theFile.close()
 
         theReleaseList = getCommandOutput("echo `scramv1 l -a | awk '$1==\"CMSSW\" && /CMSSW_"+opts.inputrelease+"/ { print $2  }' | sort`")
-        #print theReleaseList
+        #print(theReleaseList)
         theRelease = theReleaseList.split()[-1]
-        #print theRelease
+        #print(theRelease)
 
         commands = []
         commands.append("#!/bin/tcsh")
@@ -256,11 +256,11 @@ def main():
 
         theExecutableFile = open("testing.csh", "w" )
 
-        print "-------------------------------------------------------"
-        print "Will run the following commands"
-        print "-------------------------------------------------------"     
+        print("-------------------------------------------------------")
+        print("Will run the following commands")
+        print("-------------------------------------------------------")     
         for command in commands:
-            print command
+            print(command)
             theExecutableFile.write(command+"\n")
 
         theExecutableFile.close()

ValidateHLTMenu/ALCAMenuChecker.py

@@ -21,8 +21,8 @@
 L1MenuName="l1menu.xml"
 
 # Parse the HLT menu
-### Usually, blindly executing an external file is a security hazard... 
-execfile(HLTMenuName)
+### Usually, blindly executing an external file is a security hazard...
+exec(open(HLTMenuName).read())  # Using exec() - Python 3 compatible
 
 # Parse the L1 menu. Notice that here we are parsing the version that is
 # usually available from the Twiki:
@@ -34,7 +34,7 @@
 for algo in root.findall('algorithm'):
     listOfAvailableSeeds.append(algo[0].text)
 
-print process.process
+print(process.process)
 pathnames = process.paths.viewkeys()
 
 pathsVsSeeds = dict()
@@ -51,7 +51,7 @@ def splitL1seeds(fullSeed):
 # 0) Get the number of HLT columns
 numberOfHLTColumns = len(process.PrescaleService.lvl1Labels)
 HLTColumnIndexes = range(0,numberOfHLTColumns)
-print "HLT menu has",numberOfHLTColumns,"columns"
+print("HLT menu has",numberOfHLTColumns,"columns")
 
 # 1) Make the map of path vs list of seeds
 for path in process.paths:
@@ -66,7 +66,7 @@ def splitL1seeds(fullSeed):
             pathsVsSeeds[thePath.label()] = list()
 
 # 2) Make the map of path vs prescales
-print "-"*64
+print("-"*64)
 for i in process.PrescaleService.prescaleTable:
 	# We don't want the Output paths that may be here
 	if "Output" in i.pathName.value(): 
@@ -78,14 +78,14 @@ def splitL1seeds(fullSeed):
 	if pathName not in pathPrescales.keys():
 		pathPrescales[pathName] = [1]*numberOfHLTColumns
 		if not("Calibration" in pathName or "HLTriggerFirstPath" in pathName or "HLTriggerFinalPath" in pathName):
-			print RED+'WARNING:'+RESET,pathName,"has no defined HLT prescales"
+			print(RED+'WARNING:'+RESET,pathName,"has no defined HLT prescales")
 	if len(pathsVsSeeds[pathName]) == 0:
 		L1pathPrescales[pathName] = [1]*numberOfHLTColumns
 
 # NOW come the AlCa checks proper
 
 # 1) Do I have all the AlCa datasets?
-print "-"*64
+print("-"*64)
 datasetNames = process.datasets._Parameterizable__parameterNames
 mandatoryDatasetsAndPaths = {"ExpressPhysics":["HLT_IsoMu20_v*",
                                                "HLT_IsoMu24_v*",
@@ -122,20 +122,20 @@ def splitL1seeds(fullSeed):
 
 for mds in mandatoryDatasets:	
     if mds in datasetNames:
-        print row_format.format(mds),GREEN+"PRESENT"+RESET		
+        print(row_format.format(mds),GREEN+"PRESENT"+RESET)		
         presentMandatoryDatasets.append(mds)
     else:
-        print row_format.format(mds),RED+"ABSENT"+RESET
+        print(row_format.format(mds),RED+"ABSENT"+RESET)
         presentMandatoryDatasets.sort()
 
 # 2) Do the datasets have all paths they should have?
-print "-"*64
+print("-"*64)
 for mds in presentMandatoryDatasets:
     theDataset = getattr(process.datasets,mds)
     for requestedPath in mandatoryDatasetsAndPaths[mds]:
         pathIsPresent = False
         if len(fnmatch.filter(theDataset,requestedPath)) == 0:
-            print row_format.format(mds),row_format2.format(requestedPath),RED+"ABSENT"+RESET
+            print(row_format.format(mds),row_format2.format(requestedPath),RED+"ABSENT"+RESET)
         # Do the paths have at least one L1 seed available in the menu?    
         for matchingPath in fnmatch.filter(theDataset,requestedPath):
             hasL1Seed = False
@@ -146,17 +146,17 @@ def splitL1seeds(fullSeed):
                 if seed in listOfAvailableSeeds:
                     hasL1Seed = True
             if not hasL1Seed:
-                print row_format.format(mds),row_format2.format(matchingPath),GREEN+"PRESENT"+RESET,"but ",RED+"NO L1 SEED"+RESET
+                print(row_format.format(mds),row_format2.format(matchingPath),GREEN+"PRESENT"+RESET,"but ",RED+"NO L1 SEED"+RESET)
             elif hasL1Seed:
-                print row_format.format(mds),row_format2.format(matchingPath),GREEN+"PRESENT"+RESET,"and ",GREEN+"HAS L1 SEED"+RESET
+                print(row_format.format(mds),row_format2.format(matchingPath),GREEN+"PRESENT"+RESET,"and ",GREEN+"HAS L1 SEED"+RESET)
 
 
 # 3) Check the smart prescales of the Express Datasets:
-print "-"*64
+print("-"*64)
 for mds in presentMandatoryDatasets:
 	if "Express" in mds:
 		normalizedDSName = mds.replace("Physics","")
-		print BOLD+normalizedDSName+RESET
+		print(BOLD+normalizedDSName+RESET)
 		smartPrescales = getattr(process,"hltPre"+normalizedDSName+"OutputSmart")
-		print smartPrescales.triggerConditions
-		print "\n"
+		print(smartPrescales.triggerConditions)
+		print("\n")

ValidateHLTMenu/ValidateHLTMenu.py

@@ -114,7 +114,7 @@ def find_hlt_path(PrimaryDataset, HLTpath, HLTMenu, output_file):
         return True, matching_triggers
 
 
-def find_pd(primary_dataset):
+def find_pd(primary_dataset, HLTMenu):
     """
     """
     pd_match = ""
@@ -204,7 +204,7 @@ def analyze(AlCaRecoMatrix, AlCaRecoTriggerBits, configuration, HLTMenu, hlt_men
                             "MatchingTriggers": matching_triggers
                         })            
             else:
-                pd_match, pd_unmatch = find_pd(primary_dataset)
+                pd_match, pd_unmatch = find_pd(primary_dataset, HLTMenu)
                 output[TriggerBitKey]["PDMatch"] = pd_match
                 output[TriggerBitKey]["PDUnMatch"] = pd_unmatch
                 if pd_match == "":

ValidateHLTMenu/testAlCaRecoMatrix.py

@@ -102,7 +102,7 @@
 # Parse the HLT menu
 ### Usually, blindly executing an external file is a security hazard... 
 print("Parsing HLT menu...")
-execfile(HLTMenuName)
+exec(open(HLTMenuName).read())  # Using exec() - Python 3 compatible
 
 print("List of datasets:")
 print(sorted(process.datasets._Parameterizable__parameterNames))