test/scripts/AnalyzeLumiScan.py

0001 #!/usr/bin/env python3
0002 #____________________________________________________________
0003 #
0004 #  AnalyzeLumiScan
0005 #
0006 # A very simple way to run beam fit for lumi scan
0007 # Running with RecoVertex/BeamSpotProducer package
0008 # Create log and beam fit txt files in the dir named <Run number>
0009 #
0010 # Geng-yuan Jeng
0011 # Geng-yuan.Jeng@cern.ch
0012 #
0013 # Fermilab, 2009
0014 #
0015 #____________________________________________________________
0016
0017 from __future__ import print_function
0018 from builtins import range
0019 import sys,os,re,string
0020
0021 ##dataset = "ExpressPhysics/BeamCommissioning09-Express-v2/FEVT"
0022 dataset = "/MinimumBias/BeamCommissioning09-Dec19thReReco_341_v1/RECO"
0023 def get_list_files(run,ls1,ls2,mode):
0024     lfiles = []
0025     ## DBS query
0026     dbsquery = "dbs search --noheader --query=\"find file where dataset="+dataset
0027     dbsquery += " and run="+str(run)+" and lumi>="+str(ls1)+" and lumi<="+str(ls2)+"\" > tmplist.txt"
0028 ## For running on ExpressPhysics datasets ordered by lumi, uncomment the following line and comment out the previous one
0029 ##    dbsquery += " and run="+str(run)+" and lumi>="+str(ls1)+" and lumi<="+str(ls2)+" order by lumi asc\" > tmplist.txt"
0030     #print dbsquery
0031     os.system(dbsquery)
0032     n=0
0033     fileHandle = open("tmplist.txt")
0034     lineList = fileHandle.readlines()
0035     nfiles = len(lineList)-1
0036
0037     lineList = map(string.strip, lineList)
0038     if mode == 0: ## caf
0039         prefix=""
0040     elif mode == 1: ## lxplus
0041         prefix="rfio:/castor/cern.ch/cms"
0042     elif mode == 2: ## cmslpc
0043     prefix="dcache:/pnfs/cms/WAX/11"
0044     else:
0045         print("Mode = "+str(mode)+" is not supported")
0046
0047     for f in lineList:
0048         #print n
0049         if f.find("store") != -1:
0050             #print f
0051             if n < nfiles:
0052                 lfiles.append("'" + prefix + f + "',\n")
0053             else:
0054                 lfiles.append("'" + prefix + f + "'\n")
0055         n=n+1
0056     fileHandle.close()
0057     os.system("rm tmplist.txt")
0058     return lfiles
0059
0060 def main():
0061
0062     if len(sys.argv) < 4:
0063         print("\n [Usage] python AnalyzeLumiScan.py <LumiScanLists.txt> <caf/lxplus/cmslpc> <local/batch>")
0064         sys.exit()
0065
0066     cfidir = "../../python/"
0067     cfifilepath = "RecoVertex.BeamSpotProducer."
0068
0069     lumilistfile = sys.argv[1]
0070     mode = sys.argv[2]
0071     jobmode = sys.argv[3]
0072     if mode == "caf":
0073         fmode = 0
0074     elif mode == "lxplus":
0075         fmode = 1
0076     elif mode == "cmslpc":
0077     fmode = 2
0078     jobmode = "local" ## temporary
0079     else:
0080         print("Mode not supported")
0081         sys.exit()
0082
0083     if jobmode != "local" and jobmode != "batch":
0084         print("Jobe mode not supported")
0085         sys.exit()
0086     runinfofile = open(lumilistfile,"r")
0087     runinfolist = runinfofile.readlines()
0088     runsinfo = {}
0089
0090     for line in runinfolist:
0091         npos=0
0092         for i in line.split():
0093             npos+=1
0094             if npos == 1:
0095                 run="Run"+str(i)+"/"
0096             else:
0097                 if i.lower() == "max":
0098                     dbsquery ="dbs search --noheader --query=\"find max(lumi) where dataset="+dataset
0099                     dbsquery += " and run ="+run[3:len(run)-1]+"\""
0100 ##                    print dbsquery
0101                     i = os.popen(dbsquery).read()
0102                 runsinfo.setdefault(run,[]).append(int(i))
0103 ##    print runsinfo
0104
0105     for i in range(len(runsinfo)):
0106         d=runsinfo.keys()[i]
0107         if os.path.exists(d):
0108             print("Directory \""+d[:len(d)-1]+"\" exists!!!")
0109             sys.exit() ## remove for test
0110
0111     for i in range(len(runsinfo)):
0112         d=runsinfo.keys()[i]
0113         os.system("mkdir -p "+d[:len(d)-1])
0114         print("Output and log files will be saved to: ", end=' ')
0115         print(d[:len(d)-1])
0116
0117         ## Create input cfi files according to run and lumi sections
0118         lumilist=runsinfo.get(d)
0119         for j in range((len(lumilist)+1)/2):
0120             files = get_list_files(int(d[3:len(d)-1]),lumilist[j*2],lumilist[j*2+1],fmode)
0121             if lumilist[j*2] < 10:
0122                 minlumi = "00"+str(lumilist[j*2])
0123             elif lumilist[j*2] < 100:
0124                 minlumi = "0"+str(lumilist[j*2])
0125             elif lumilist[j*2] < 1000:
0126                 minlumi = str(lumilist[j*2])
0127             else:
0128                 print("Lumi range greater than 1000!!!")
0129                 sys.exit()
0130
0131             if lumilist[j*2+1] < 10:
0132                 maxlumi = "00"+str(lumilist[j*2+1])
0133             elif lumilist[j*2+1] < 100:
0134                 maxlumi = "0"+str(lumilist[j*2+1])
0135             elif lumilist[j*2+1] < 1000:
0136                 maxlumi = str(lumilist[j*2+1])
0137             else:
0138                 print("Lumi range greater than 1000!!!")
0139                 sys.exit()
0140
0141             tagName = d[:len(d)-1]+"LS"+minlumi+"to"+maxlumi
0142 ##            print tagName
0143
0144             fouttmp = open("Input_template_cfi.py")
0145             cfiname = cfidir+"LumiScan_"+tagName+"_cfi.py"
0146             fout = open(cfiname,"w")
0147
0148             for line in fouttmp:
0149                 if line.find("INPUTFILES")!=-1:
0150                     for f in files:
0151                         fout.write(f)
0152
0153                 if not "INPUTFILES" in line:
0154                     fout.write(line)
0155
0156             fout.close()
0157
0158             ## Copy cfi file to run directories
0159             os.system("cp "+cfiname+" "+d)
0160             ## Create cfg files for cmsRun
0161             cfinametag = cfifilepath+"LumiScan_"+tagName+"_cfi"
0162             #print cfinametag
0163             lumirange = d[3:len(d)-1]+":"+str(lumilist[j*2])+"-"+d[3:len(d)-1]+":"+str(lumilist[j*2+1])
0164             #print lumirange
0165             asciioutfile = d+"LumiScan_"+tagName+".txt"
0166             #print asciioutfile
0167             treeoutfile = d+"LumiScan_"+tagName+".root"
0168             #print treeoutfile
0169             replacetag = [('INPUT_FILE',cfinametag),
0170                           ('LUMIRANGE',lumirange),
0171                           ('ASCIIFILE',asciioutfile),
0172                           ('OUTPUTFILE',treeoutfile)
0173                           ]
0174
0175             tmpcfgfile = open("analyze_lumiscan_template_cfg.py")
0176             cfgtagname = "analyze_lumiscan_"+tagName+"_cfg.py"
0177             newcfgfile = open(cfgtagname,"w")
0178             for line in tmpcfgfile:
0179                 for itag in replacetag:
0180                     line = line.replace(itag[0],itag[1])
0181
0182                 newcfgfile.write(line)
0183
0184             newcfgfile.close()
0185
0186             os.system("mv "+cfgtagname+" "+d)
0187             if jobmode == "local":
0188                 runjobcmd = "cmsRun "+d+cfgtagname+" >& "+d+"LumiScan_"+tagName+".log &"
0189                 print(runjobcmd)
0190                 os.system(runjobcmd)
0191
0192             ## Create job and submitting to batch
0193             if jobmode == "batch":
0194                 workdir = os.environ['PWD']
0195 ##                print workdir
0196                 runlogtag = d+"LumiScan_"+tagName+".log"
0197                 replacetag1 = [('PWD',workdir),
0198                                ('CFGFILE',d+cfgtagname),
0199                                ('CFGRUNLOG',runlogtag)
0200                                ]
0201                 tmpbjobfile = open("AnalyzeLumiScanJob_template.sh","r")
0202                 bjobName = "AnalyzeLumiScanJob_"+tagName+".sh"
0203                 newbjobfile = open(bjobName,'w')
0204
0205                 for line in tmpbjobfile.readlines():
0206                     for itag in replacetag1:
0207                         line = line.replace(itag[0],itag[1])
0208                     newbjobfile.write(line)
0209                 newbjobfile.write(line)
0210                 os.system("chmod +x "+bjobName)
0211                 if os.environ['SCRAM_ARCH'] == "slc5_ia32_gcc434":
0212                     submitjobcmd = "bsub -q 8nh -R \"type=SLC5_64\" "+bjobName
0213                     print(submitjobcmd)
0214                 else:
0215                     submitjobcmd = "bsub -q 8nh "+bjobName
0216                     print(submitjobcmd)
0217
0218                 os.system(submitjobcmd)
0219
0220     print("End of submitting jobs")
0221
0222 #_________________________________
0223 if __name__ =='__main__':
0224     sys.exit(main())