test/scripts/AnalyzeLumiScan.py

0001 #!/usr/bin/env python3
0002 #____________________________________________________________
0003 #
0004 #  AnalyzeLumiScan
0005 #
0006 # A very simple way to run beam fit for lumi scan
0007 # Running with RecoVertex/BeamSpotProducer package
0008 # Create log and beam fit txt files in the dir named <Run number>
0009 #
0010 # Geng-yuan Jeng
0011 # Geng-yuan.Jeng@cern.ch
0012 #
0013 # Fermilab, 2009
0014 #
0015 #____________________________________________________________
0016
0017 from builtins import range
0018 import sys,os,re,string
0019
0020 ##dataset = "ExpressPhysics/BeamCommissioning09-Express-v2/FEVT"
0021 dataset = "/MinimumBias/BeamCommissioning09-Dec19thReReco_341_v1/RECO"
0022 def get_list_files(run,ls1,ls2,mode):
0023     lfiles = []
0024     ## DBS query
0025     dbsquery = "dbs search --noheader --query=\"find file where dataset="+dataset
0026     dbsquery += " and run="+str(run)+" and lumi>="+str(ls1)+" and lumi<="+str(ls2)+"\" > tmplist.txt"
0027 ## For running on ExpressPhysics datasets ordered by lumi, uncomment the following line and comment out the previous one
0028 ##    dbsquery += " and run="+str(run)+" and lumi>="+str(ls1)+" and lumi<="+str(ls2)+" order by lumi asc\" > tmplist.txt"
0029     #print dbsquery
0030     os.system(dbsquery)
0031     n=0
0032     fileHandle = open("tmplist.txt")
0033     lineList = fileHandle.readlines()
0034     nfiles = len(lineList)-1
0035
0036     lineList = map(string.strip, lineList)
0037     if mode == 0: ## caf
0038         prefix=""
0039     elif mode == 1: ## lxplus
0040         prefix="rfio:/castor/cern.ch/cms"
0041     elif mode == 2: ## cmslpc
0042     prefix="dcache:/pnfs/cms/WAX/11"
0043     else:
0044         print("Mode = "+str(mode)+" is not supported")
0045
0046     for f in lineList:
0047         #print n
0048         if f.find("store") != -1:
0049             #print f
0050             if n < nfiles:
0051                 lfiles.append("'" + prefix + f + "',\n")
0052             else:
0053                 lfiles.append("'" + prefix + f + "'\n")
0054         n=n+1
0055     fileHandle.close()
0056     os.system("rm tmplist.txt")
0057     return lfiles
0058
0059 def main():
0060
0061     if len(sys.argv) < 4:
0062         print("\n [Usage] python AnalyzeLumiScan.py <LumiScanLists.txt> <caf/lxplus/cmslpc> <local/batch>")
0063         sys.exit()
0064
0065     cfidir = "../../python/"
0066     cfifilepath = "RecoVertex.BeamSpotProducer."
0067
0068     lumilistfile = sys.argv[1]
0069     mode = sys.argv[2]
0070     jobmode = sys.argv[3]
0071     if mode == "caf":
0072         fmode = 0
0073     elif mode == "lxplus":
0074         fmode = 1
0075     elif mode == "cmslpc":
0076     fmode = 2
0077     jobmode = "local" ## temporary
0078     else:
0079         print("Mode not supported")
0080         sys.exit()
0081
0082     if jobmode != "local" and jobmode != "batch":
0083         print("Jobe mode not supported")
0084         sys.exit()
0085     runinfofile = open(lumilistfile,"r")
0086     runinfolist = runinfofile.readlines()
0087     runsinfo = {}
0088
0089     for line in runinfolist:
0090         npos=0
0091         for i in line.split():
0092             npos+=1
0093             if npos == 1:
0094                 run="Run"+str(i)+"/"
0095             else:
0096                 if i.lower() == "max":
0097                     dbsquery ="dbs search --noheader --query=\"find max(lumi) where dataset="+dataset
0098                     dbsquery += " and run ="+run[3:len(run)-1]+"\""
0099 ##                    print dbsquery
0100                     i = os.popen(dbsquery).read()
0101                 runsinfo.setdefault(run,[]).append(int(i))
0102 ##    print runsinfo
0103
0104     for i in range(len(runsinfo)):
0105         d=runsinfo.keys()[i]
0106         if os.path.exists(d):
0107             print("Directory \""+d[:len(d)-1]+"\" exists!!!")
0108             sys.exit() ## remove for test
0109
0110     for i in range(len(runsinfo)):
0111         d=runsinfo.keys()[i]
0112         os.system("mkdir -p "+d[:len(d)-1])
0113         print("Output and log files will be saved to: ", end=' ')
0114         print(d[:len(d)-1])
0115
0116         ## Create input cfi files according to run and lumi sections
0117         lumilist=runsinfo.get(d)
0118         for j in range((len(lumilist)+1)/2):
0119             files = get_list_files(int(d[3:len(d)-1]),lumilist[j*2],lumilist[j*2+1],fmode)
0120             if lumilist[j*2] < 10:
0121                 minlumi = "00"+str(lumilist[j*2])
0122             elif lumilist[j*2] < 100:
0123                 minlumi = "0"+str(lumilist[j*2])
0124             elif lumilist[j*2] < 1000:
0125                 minlumi = str(lumilist[j*2])
0126             else:
0127                 print("Lumi range greater than 1000!!!")
0128                 sys.exit()
0129
0130             if lumilist[j*2+1] < 10:
0131                 maxlumi = "00"+str(lumilist[j*2+1])
0132             elif lumilist[j*2+1] < 100:
0133                 maxlumi = "0"+str(lumilist[j*2+1])
0134             elif lumilist[j*2+1] < 1000:
0135                 maxlumi = str(lumilist[j*2+1])
0136             else:
0137                 print("Lumi range greater than 1000!!!")
0138                 sys.exit()
0139
0140             tagName = d[:len(d)-1]+"LS"+minlumi+"to"+maxlumi
0141 ##            print tagName
0142
0143             fouttmp = open("Input_template_cfi.py")
0144             cfiname = cfidir+"LumiScan_"+tagName+"_cfi.py"
0145             fout = open(cfiname,"w")
0146
0147             for line in fouttmp:
0148                 if line.find("INPUTFILES")!=-1:
0149                     for f in files:
0150                         fout.write(f)
0151
0152                 if not "INPUTFILES" in line:
0153                     fout.write(line)
0154
0155             fout.close()
0156
0157             ## Copy cfi file to run directories
0158             os.system("cp "+cfiname+" "+d)
0159             ## Create cfg files for cmsRun
0160             cfinametag = cfifilepath+"LumiScan_"+tagName+"_cfi"
0161             #print cfinametag
0162             lumirange = d[3:len(d)-1]+":"+str(lumilist[j*2])+"-"+d[3:len(d)-1]+":"+str(lumilist[j*2+1])
0163             #print lumirange
0164             asciioutfile = d+"LumiScan_"+tagName+".txt"
0165             #print asciioutfile
0166             treeoutfile = d+"LumiScan_"+tagName+".root"
0167             #print treeoutfile
0168             replacetag = [('INPUT_FILE',cfinametag),
0169                           ('LUMIRANGE',lumirange),
0170                           ('ASCIIFILE',asciioutfile),
0171                           ('OUTPUTFILE',treeoutfile)
0172                           ]
0173
0174             tmpcfgfile = open("analyze_lumiscan_template_cfg.py")
0175             cfgtagname = "analyze_lumiscan_"+tagName+"_cfg.py"
0176             newcfgfile = open(cfgtagname,"w")
0177             for line in tmpcfgfile:
0178                 for itag in replacetag:
0179                     line = line.replace(itag[0],itag[1])
0180
0181                 newcfgfile.write(line)
0182
0183             newcfgfile.close()
0184
0185             os.system("mv "+cfgtagname+" "+d)
0186             if jobmode == "local":
0187                 runjobcmd = "cmsRun "+d+cfgtagname+" >& "+d+"LumiScan_"+tagName+".log &"
0188                 print(runjobcmd)
0189                 os.system(runjobcmd)
0190
0191             ## Create job and submitting to batch
0192             if jobmode == "batch":
0193                 workdir = os.environ['PWD']
0194 ##                print workdir
0195                 runlogtag = d+"LumiScan_"+tagName+".log"
0196                 replacetag1 = [('PWD',workdir),
0197                                ('CFGFILE',d+cfgtagname),
0198                                ('CFGRUNLOG',runlogtag)
0199                                ]
0200                 tmpbjobfile = open("AnalyzeLumiScanJob_template.sh","r")
0201                 bjobName = "AnalyzeLumiScanJob_"+tagName+".sh"
0202                 newbjobfile = open(bjobName,'w')
0203
0204                 for line in tmpbjobfile.readlines():
0205                     for itag in replacetag1:
0206                         line = line.replace(itag[0],itag[1])
0207                     newbjobfile.write(line)
0208                 newbjobfile.write(line)
0209                 os.system("chmod +x "+bjobName)
0210                 if os.environ['SCRAM_ARCH'] == "slc5_ia32_gcc434":
0211                     submitjobcmd = "bsub -q 8nh -R \"type=SLC5_64\" "+bjobName
0212                     print(submitjobcmd)
0213                 else:
0214                     submitjobcmd = "bsub -q 8nh "+bjobName
0215                     print(submitjobcmd)
0216
0217                 os.system(submitjobcmd)
0218
0219     print("End of submitting jobs")
0220
0221 #_________________________________
0222 if __name__ =='__main__':
0223     sys.exit(main())