CalibCalo/test/dohistos.py

0001 #! /usr/bin/env python3
0002
0003 import os
0004 import sys
0005 import errno
0006
0007 #print error/help message and exit
0008 def help_message():
0009     print("Usage:\n\
0010 dohistos [folder_name] [options] -v versions_to_compare -f files_to_compare\n\
0011 Versions and files must be whitespace separated.\n\
0012 If no folder is specified the pwd will be used.\n\
0013 folder_name, if specified, must be the first argument.\n\
0014 Options:\n\
0015 --outfile out.root    specify the output root file name\n\
0016 -h    prints this message\n\
0017 --cvs    uses cvs access to retrieve edm2me_cfg.py\n\
0018 -a    uses anonymous cvs access (only for debug purposes)\n\
0019 --dbs    use dbs access to retrieve file paths, edit directly the source code to modify the query, while in --dbs mode don't use -f option\n\
0020 --condition [IDEAL,STARTUP]    permits to specify the conditions when in dbs mode; if not specified IDEAL is default; must be coherent with --query option if specified\n\
0021 -n [1000]   specify the number of events\n\
0022 --like   additional like statement in the dbs query to narrow the number of dataset returned\n\
0023 Example:\n\
0024 ./dohistos.py Folder01 -v CMSSW_X_Y_Z CMSSW_J_K_W -f file:/file1.root file:/file2.root")
0025     sys.exit()
0026 #custom query is at the moment not useful:
0027 #--query    with --dbs option specify the query passed to dbs cli (with quotes)\n\
0028
0029 #run command in the command line with specified environment
0030 def runcmd(envir,program,*args):
0031     pid=os.fork()
0032     if not pid:
0033         os.execvpe(program,(program,)+args,envir)
0034     return os.wait()[0]
0035
0036 #print the help message
0037 if "-h" in sys.argv or "-help" in sys.argv or "--help" in sys.argv:
0038     help_message()
0039
0040 #make the working directory (if specified) and move there
0041 if len(sys.argv)>1:
0042     if not sys.argv[1][0]=="-":
0043         name=sys.argv[1]
0044         try:
0045             os.mkdir(name)
0046         except OSError as inst:
0047             if inst.errno==errno.EEXIST:
0048                 print("Warning: the specified working folder already exist")
0049         os.chdir(name)
0050 else: help_message()
0051
0052 #read and parse the input
0053 state="n"
0054 like_query=""
0055 num_evts="1000"
0056 cvs=False
0057 dbs=False
0058 use_manual_num=False
0059 anon=False#only for debug purposes
0060 conditions='IDEAL_31X'
0061 conditions_file='IDEAL'
0062 #query_list=[]    #not useful
0063 ver=[]
0064 fil=[]
0065 out_root="histo.root"
0066 #used state letters (please keep updated): cflnqrv
0067 for arg in sys.argv:
0068     if arg=="-v" or arg=="-V":
0069         state="v"
0070     elif arg=="-f" or arg=="-F":
0071         state="f"
0072     elif arg=="--outfile":
0073         state="r"
0074     elif arg=="--conditions":
0075         state="c"
0076     elif arg=="-n":
0077         state="n"
0078     #elif arg=="--query":    #not useful
0079     #    state="q"
0080     elif arg=="--cvs":
0081         cvs=True
0082     elif arg=="--dbs":
0083         dbs=True
0084     elif arg=="-a":#only for debug purposes
0085         anon=True
0086     elif arg=="--like":
0087         state="l"
0088 ############################################## state handling
0089     elif state=="v":
0090         ver.append(arg)
0091     elif state=="f":
0092         fil.append(arg)
0093     elif state=="r":
0094         out_root=arg
0095     elif state=="l":
0096         like_query=arg
0097     elif state=="c":
0098         conditions=arg
0099         usn=0
0100         for ncondt,condt in enumerate(arg):
0101             if condt=='_':
0102                 usn=ncondt
0103                 break
0104         conditions_file=conditions[:usn]
0105     elif state=="n":
0106         num_evts=arg
0107         use_manual_num=True
0108     #elif state=="q":    #not useful
0109     #    query_list.append(arg)
0110
0111 #check consistency of -f and --dbs
0112 if len(fil)>0 and dbs:
0113     print("when using --dbs option, -f option is not needed")
0114     help_message()
0115
0116 ###dbs query to retrieve the data with option --dbs
0117 ###|||||||||||||||||||||||||||||||||||||||||||||||||
0118 dbsstr='python $DBSCMD_HOME/dbsCommandLine.py -c search --query="find dataset where phygrp=RelVal and primds=RelValMinBias and dataset.tier=GEN-SIM-DIGI-RAW-HLTDEBUG and file.name like *'+conditions+'* and file.name like *'+like_query+'* and file.release='
0119 #dbsstr='dbs -c search --query="find dataset where phygrp=RelVal and primds=RelValMinBias and dataset.tier=GEN-SIM-DIGI-RAW-HLTDEBUG and file.name like *'+conditions+'* and file.release='
0120 ###|||||||||||||||||||||||||||||||||||||||||||||||||
0121 dbsdataset=[]
0122 dbsfile=[]
0123 nevt=[]
0124 nevent=0
0125
0126 #create folders and generate files
0127 for nv,v in enumerate(ver):
0128     os.system("scramv1 project CMSSW "+v)
0129     os.chdir(v)
0130     env=os.popen("scramv1 runtime -sh","r")
0131     environment=os.environ
0132     for l in env.readlines():
0133         try:
0134             variable,value=l[7:len(l)-3].strip().split("=",1)
0135             environment[variable]=value[1:]
0136         except ValueError:
0137             print("Warning: environment variable problem")
0138     env.close()
0139     if cvs:
0140         if anon:#only for debug purposes, works only in cmsfarm
0141             os.system("eval `scramv1 runtime -sh`; source /cms-sw/slc4_ia32_gcc345/cms/cms-cvs-utils/1.0/bin/cmscvsroot.sh CMSSW; cvs login; addpkg DQMOffline/CalibCalo")
0142         else:
0143             runcmd(environment,"addpkg","DQMOffline/CalibCalo")
0144
0145 #dbs code
0146     if dbs:
0147         dbsdataset=[]
0148         dbsfile=[]
0149         nevt=[]
0150         #searching the required dataset
0151         inifil=False
0152         ris=os.popen(dbsstr+v+'"')
0153         for lnris in ris.readlines():
0154             print(lnris)
0155             if inifil:
0156                 dbsdataset.append(lnris)
0157             else:
0158                 #if lnris[:3]=="___":
0159                 if lnris[:3]=="---":
0160                     inifil=True
0161         ris.close()
0162         dbsdataset=dbsdataset[2:]
0163         dbsdataset[0]=dbsdataset[0][0:-1]
0164         for lnris2 in dbsdataset:
0165             print(lnris2)
0166         if len(dbsdataset)>1 or len(dbsdataset)==0:
0167             #print dbsdataset
0168             print("dbs search returned ",len(dbsdataset)," records, please modify the query so only one dataset is returned")
0169             sys.exit()
0170         else:
0171             #extracting the file names relative to the selected dataset
0172             inifil=False
0173             ris=os.popen('python $DBSCMD_HOME/dbsCommandLine.py -c search --query="find file where dataset like *'+dbsdataset[0]+'*"')
0174             for lnris in ris.readlines():
0175                 if inifil:
0176                     dbsfile.append(lnris)
0177                 else:
0178                     if lnris[:3]=="---":
0179                         inifil=True
0180             ris.close()
0181             dbsfile=dbsfile[2:]
0182             for dbsfn,dbsf in enumerate(dbsfile):
0183                 dbsfile[dbsfn]=dbsfile[dbsfn][:-1]
0184
0185             #extracting the total number of events    #not very useful at the moment, it is better to use manual extraction
0186             #if not use_manual_num:
0187             #    for dbsf in dbsfile:
0188             #        inifil=False
0189             #        ris=os.popen('python $DBSCMD_HOME/dbsCommandLine.py -c search --query="find file.numevents where file like *'+dbsf+'*"')
0190             #        for lnris in ris:
0191             #            if inifil:
0192             #                nevt.append(lnris)
0193             #            else:
0194             #                if lnris[:3]=="___":
0195             #                    inifil=True
0196             #        nevt.pop()
0197             #        ris.close()
0198             #    for nevtn,nevte in nevt:
0199             #        nevt[nevtn]=int(nevt[nevtn][:-2])
0200             #        nevent
0201             #    for nevte in nevt:
0202             #
0203
0204     #for f in fil: remember indentation if uncommenting this
0205     if not dbs:
0206         runcmd(environment,"cmsDriver.py","testALCA","-s","ALCA:Configuration/StandardSequences/AlCaRecoStream_EcalCalPhiSym_cff:EcalCalPhiSym+DQM","-n",num_evts,"--filein",fil[nv],"--fileout","file:dqm.root","--eventcontent","FEVT","--conditions","FrontierConditions_GlobalTag,"+conditions+"::All")#,"--no_exec")
0207     else:
0208         sfl=""
0209         for fl in dbsfile:
0210             sfl=sfl+','+fl
0211         sfl=sfl[1:]
0212         runcmd(environment,"cmsDriver.py","testALCA","-s","ALCA:Configuration/StandardSequences/AlCaRecoStream_EcalCalPhiSym_cff:EcalCalPhiSym+DQM","-n",num_evts,"--filein",sfl,"--fileout","file:dqm.root","--eventcontent","FEVT","--conditions","FrontierConditions_GlobalTag,"+conditions+"::All","--no_exec")
0213         alcareco=open("testALCA_ALCA_"+conditions_file+".py",'r')
0214         alcarecoln=alcareco.readlines()
0215         alcareco.close()
0216         arnum=0
0217         for arln,arl in enumerate(alcarecoln):
0218             if sfl in arl:
0219                 arnum=arln
0220         alcarecoln[arnum]=alcarecoln[arnum].replace(",","','")
0221         alcareco=open("testALCA_ALCA_"+conditions_file+".py",'w')
0222         for arln in alcarecoln:
0223             alcareco.write(arln)
0224         alcareco.close()
0225         runcmd(environment,"cmsRun","testALCA_ALCA_"+conditions_file+".py")
0226     os.system("mv ALCARECOEcalCalPhiSym.root dqm.root")
0227     if cvs:
0228         runcmd(environment,"cmsRun","src/DQMOffline/CalibCalo/test/edm2me_cfg.py")
0229     else:
0230         runcmd(environment,"cmsRun",environment["CMSSW_RELEASE_BASE"]+"/src/DQMOffline/CalibCalo/test/edm2me_cfg.py")
0231     os.system("mv DQM_V0001_R000000001__A__B__C.root "+out_root)
0232     os.system("rm dqm.root")
0233     os.chdir("../")