CalibCalo/test/dohistos.py

0001 #! /usr/bin/env python3
0002
0003 from __future__ import print_function
0004 import os
0005 import sys
0006 import errno
0007
0008 #print error/help message and exit
0009 def help_message():
0010     print("Usage:\n\
0011 dohistos [folder_name] [options] -v versions_to_compare -f files_to_compare\n\
0012 Versions and files must be whitespace separated.\n\
0013 If no folder is specified the pwd will be used.\n\
0014 folder_name, if specified, must be the first argument.\n\
0015 Options:\n\
0016 --outfile out.root    specify the output root file name\n\
0017 -h    prints this message\n\
0018 --cvs    uses cvs access to retrieve edm2me_cfg.py\n\
0019 -a    uses anonymous cvs access (only for debug purposes)\n\
0020 --dbs    use dbs access to retrieve file paths, edit directly the source code to modify the query, while in --dbs mode don't use -f option\n\
0021 --condition [IDEAL,STARTUP]    permits to specify the conditions when in dbs mode; if not specified IDEAL is default; must be coherent with --query option if specified\n\
0022 -n [1000]   specify the number of events\n\
0023 --like   additional like statement in the dbs query to narrow the number of dataset returned\n\
0024 Example:\n\
0025 ./dohistos.py Folder01 -v CMSSW_X_Y_Z CMSSW_J_K_W -f file:/file1.root file:/file2.root")
0026     sys.exit()
0027 #custom query is at the moment not useful:
0028 #--query    with --dbs option specify the query passed to dbs cli (with quotes)\n\
0029
0030 #run command in the command line with specified environment
0031 def runcmd(envir,program,*args):
0032     pid=os.fork()
0033     if not pid:
0034         os.execvpe(program,(program,)+args,envir)
0035     return os.wait()[0]
0036
0037 #print the help message
0038 if "-h" in sys.argv or "-help" in sys.argv or "--help" in sys.argv:
0039     help_message()
0040
0041 #make the working directory (if specified) and move there
0042 if len(sys.argv)>1:
0043     if not sys.argv[1][0]=="-":
0044         name=sys.argv[1]
0045         try:
0046             os.mkdir(name)
0047         except OSError as inst:
0048             if inst.errno==errno.EEXIST:
0049                 print("Warning: the specified working folder already exist")
0050         os.chdir(name)
0051 else: help_message()
0052
0053 #read and parse the input
0054 state="n"
0055 like_query=""
0056 num_evts="1000"
0057 cvs=False
0058 dbs=False
0059 use_manual_num=False
0060 anon=False#only for debug purposes
0061 conditions='IDEAL_31X'
0062 conditions_file='IDEAL'
0063 #query_list=[]    #not useful
0064 ver=[]
0065 fil=[]
0066 out_root="histo.root"
0067 #used state letters (please keep updated): cflnqrv
0068 for arg in sys.argv:
0069     if arg=="-v" or arg=="-V":
0070         state="v"
0071     elif arg=="-f" or arg=="-F":
0072         state="f"
0073     elif arg=="--outfile":
0074         state="r"
0075     elif arg=="--conditions":
0076         state="c"
0077     elif arg=="-n":
0078         state="n"
0079     #elif arg=="--query":    #not useful
0080     #    state="q"
0081     elif arg=="--cvs":
0082         cvs=True
0083     elif arg=="--dbs":
0084         dbs=True
0085     elif arg=="-a":#only for debug purposes
0086         anon=True
0087     elif arg=="--like":
0088         state="l"
0089 ############################################## state handling
0090     elif state=="v":
0091         ver.append(arg)
0092     elif state=="f":
0093         fil.append(arg)
0094     elif state=="r":
0095         out_root=arg
0096     elif state=="l":
0097         like_query=arg
0098     elif state=="c":
0099         conditions=arg
0100         usn=0
0101         for ncondt,condt in enumerate(arg):
0102             if condt=='_':
0103                 usn=ncondt
0104                 break
0105         conditions_file=conditions[:usn]
0106     elif state=="n":
0107         num_evts=arg
0108         use_manual_num=True
0109     #elif state=="q":    #not useful
0110     #    query_list.append(arg)
0111
0112 #check consistency of -f and --dbs
0113 if len(fil)>0 and dbs:
0114     print("when using --dbs option, -f option is not needed")
0115     help_message()
0116
0117 ###dbs query to retrieve the data with option --dbs
0118 ###|||||||||||||||||||||||||||||||||||||||||||||||||
0119 dbsstr='python $DBSCMD_HOME/dbsCommandLine.py -c search --query="find dataset where phygrp=RelVal and primds=RelValMinBias and dataset.tier=GEN-SIM-DIGI-RAW-HLTDEBUG and file.name like *'+conditions+'* and file.name like *'+like_query+'* and file.release='
0120 #dbsstr='dbs -c search --query="find dataset where phygrp=RelVal and primds=RelValMinBias and dataset.tier=GEN-SIM-DIGI-RAW-HLTDEBUG and file.name like *'+conditions+'* and file.release='
0121 ###|||||||||||||||||||||||||||||||||||||||||||||||||
0122 dbsdataset=[]
0123 dbsfile=[]
0124 nevt=[]
0125 nevent=0
0126
0127 #create folders and generate files
0128 for nv,v in enumerate(ver):
0129     os.system("scramv1 project CMSSW "+v)
0130     os.chdir(v)
0131     env=os.popen("scramv1 runtime -sh","r")
0132     environment=os.environ
0133     for l in env.readlines():
0134         try:
0135             variable,value=l[7:len(l)-3].strip().split("=",1)
0136             environment[variable]=value[1:]
0137         except ValueError:
0138             print("Warning: environment variable problem")
0139     env.close()
0140     if cvs:
0141         if anon:#only for debug purposes, works only in cmsfarm
0142             os.system("eval `scramv1 runtime -sh`; source /cms-sw/slc4_ia32_gcc345/cms/cms-cvs-utils/1.0/bin/cmscvsroot.sh CMSSW; cvs login; addpkg DQMOffline/CalibCalo")
0143         else:
0144             runcmd(environment,"addpkg","DQMOffline/CalibCalo")
0145
0146 #dbs code
0147     if dbs:
0148         dbsdataset=[]
0149         dbsfile=[]
0150         nevt=[]
0151         #searching the required dataset
0152         inifil=False
0153         ris=os.popen(dbsstr+v+'"')
0154         for lnris in ris.readlines():
0155             print(lnris)
0156             if inifil:
0157                 dbsdataset.append(lnris)
0158             else:
0159                 #if lnris[:3]=="___":
0160                 if lnris[:3]=="---":
0161                     inifil=True
0162         ris.close()
0163         dbsdataset=dbsdataset[2:]
0164         dbsdataset[0]=dbsdataset[0][0:-1]
0165         for lnris2 in dbsdataset:
0166             print(lnris2)
0167         if len(dbsdataset)>1 or len(dbsdataset)==0:
0168             #print dbsdataset
0169             print("dbs search returned ",len(dbsdataset)," records, please modify the query so only one dataset is returned")
0170             sys.exit()
0171         else:
0172             #extracting the file names relative to the selected dataset
0173             inifil=False
0174             ris=os.popen('python $DBSCMD_HOME/dbsCommandLine.py -c search --query="find file where dataset like *'+dbsdataset[0]+'*"')
0175             for lnris in ris.readlines():
0176                 if inifil:
0177                     dbsfile.append(lnris)
0178                 else:
0179                     if lnris[:3]=="---":
0180                         inifil=True
0181             ris.close()
0182             dbsfile=dbsfile[2:]
0183             for dbsfn,dbsf in enumerate(dbsfile):
0184                 dbsfile[dbsfn]=dbsfile[dbsfn][:-1]
0185
0186             #extracting the total number of events    #not very useful at the moment, it is better to use manual extraction
0187             #if not use_manual_num:
0188             #    for dbsf in dbsfile:
0189             #        inifil=False
0190             #        ris=os.popen('python $DBSCMD_HOME/dbsCommandLine.py -c search --query="find file.numevents where file like *'+dbsf+'*"')
0191             #        for lnris in ris:
0192             #            if inifil:
0193             #                nevt.append(lnris)
0194             #            else:
0195             #                if lnris[:3]=="___":
0196             #                    inifil=True
0197             #        nevt.pop()
0198             #        ris.close()
0199             #    for nevtn,nevte in nevt:
0200             #        nevt[nevtn]=int(nevt[nevtn][:-2])
0201             #        nevent
0202             #    for nevte in nevt:
0203             #
0204
0205     #for f in fil: remember indentation if uncommenting this
0206     if not dbs:
0207         runcmd(environment,"cmsDriver.py","testALCA","-s","ALCA:Configuration/StandardSequences/AlCaRecoStream_EcalCalPhiSym_cff:EcalCalPhiSym+DQM","-n",num_evts,"--filein",fil[nv],"--fileout","file:dqm.root","--eventcontent","FEVT","--conditions","FrontierConditions_GlobalTag,"+conditions+"::All")#,"--no_exec")
0208     else:
0209         sfl=""
0210         for fl in dbsfile:
0211             sfl=sfl+','+fl
0212         sfl=sfl[1:]
0213         runcmd(environment,"cmsDriver.py","testALCA","-s","ALCA:Configuration/StandardSequences/AlCaRecoStream_EcalCalPhiSym_cff:EcalCalPhiSym+DQM","-n",num_evts,"--filein",sfl,"--fileout","file:dqm.root","--eventcontent","FEVT","--conditions","FrontierConditions_GlobalTag,"+conditions+"::All","--no_exec")
0214         alcareco=open("testALCA_ALCA_"+conditions_file+".py",'r')
0215         alcarecoln=alcareco.readlines()
0216         alcareco.close()
0217         arnum=0
0218         for arln,arl in enumerate(alcarecoln):
0219             if sfl in arl:
0220                 arnum=arln
0221         alcarecoln[arnum]=alcarecoln[arnum].replace(",","','")
0222         alcareco=open("testALCA_ALCA_"+conditions_file+".py",'w')
0223         for arln in alcarecoln:
0224             alcareco.write(arln)
0225         alcareco.close()
0226         runcmd(environment,"cmsRun","testALCA_ALCA_"+conditions_file+".py")
0227     os.system("mv ALCARECOEcalCalPhiSym.root dqm.root")
0228     if cvs:
0229         runcmd(environment,"cmsRun","src/DQMOffline/CalibCalo/test/edm2me_cfg.py")
0230     else:
0231         runcmd(environment,"cmsRun",environment["CMSSW_RELEASE_BASE"]+"/src/DQMOffline/CalibCalo/test/edm2me_cfg.py")
0232     os.system("mv DQM_V0001_R000000001__A__B__C.root "+out_root)
0233     os.system("rm dqm.root")
0234     os.chdir("../")