Common/relval/doHarvest.py

0001 #!/usr/bin/env python
0002
0003 import sys
0004 import os
0005
0006 #Check arg,settings
0007
0008 if len(sys.argv) != 2 :
0009     print("""
0010     Usage: create_harvesting_py.py  <dataset>
0011     example:
0012     create_harvesting_py.py \
0013      /RelValTTbar/CMSSW_3_1_0_pre4_STARTUP_30X_v1/GEN-SIM-RECO
0014     """)
0015     sys.exit(10)
0016
0017 #Get data files of dataset to be processed
0018 if os.getenv('DBSCMD_HOME','NOTSET') == 'NOTSET' :
0019     print("dbs not set!")
0020     sys.exit(11)
0021
0022 if os.getenv('CMSSW_VERSION','NOTSET') == 'NOTSET' :
0023     print("""
0024     cmssw not set!
0025     example:
0026       cmsrel CMSSW_3_1_0_pre4
0027       cd CMSSW_3_1_0_pre4/src
0028       eval `scramv1 runtime -sh`
0029       cd -
0030     """)
0031     sys.exit(12)
0032
0033 dsetpath = sys.argv[1]
0034
0035 from DBSAPI.dbsApi import DbsApi
0036 from DBSAPI.dbsException import *
0037 from DBSAPI.dbsApiException import *
0038 from DBSAPI.dbsOptions import DbsOptionParser
0039
0040 optManager  = DbsOptionParser()
0041 (opts,args) = optManager.getOpt()
0042 api = DbsApi(opts.__dict__)
0043
0044 print("dataset: ", dsetpath)
0045 print("data files: ")
0046 for afile in api.listFiles(path=dsetpath):
0047   print("  %s" % afile['LogicalFileName'])
0048
0049 #Determine number of events/processes
0050 totnevts=0
0051 for afile in api.listFiles(path=dsetpath):
0052   totnevts += afile['NumberOfEvents']
0053 njobs = 1
0054 nevtref = 9000
0055 if totnevts > nevtref : njobs = (int) (totnevts / 9000)
0056 print("Total # events: ", totnevts, \
0057       " to be executed in ", njobs, "processes")
0058
0059
0060 #Run cmsDriver command
0061 raw_cmsdriver = "cmsDriver.py harvest -s HARVESTING:validationHarvesting --mc  --conditions FrontierConditions_GlobalTag,STARTUP_30X::All --harvesting AtJobEnd --no_exec -n -1"
0062
0063 print("executing cmsdriver command:\n\t", raw_cmsdriver)
0064
0065 os.system( '`' + raw_cmsdriver + '`' )
0066
0067
0068 #Open output py
0069 fin_name="harvest_HARVESTING_STARTUP.py"
0070 pyout_name = "harvest.py"
0071 os.system("touch " + fin_name)
0072 os.system('mv ' + fin_name + " " + pyout_name )
0073 pyout = open(pyout_name, 'a')
0074
0075 #Added to py config: input, output file name, dqm settings
0076 pyout.write("\n\n##additions to cmsDriver output \n")
0077 pyout.write("process.dqmSaver.workflow = '" + dsetpath + "'\n")
0078 pyout.write("process.source.fileNames = cms.untracked.vstring(\n")
0079
0080 for afile in api.listFiles(path=dsetpath):
0081     pyout.write("  '%s',\n" % afile['LogicalFileName'])
0082
0083 pyout.write(")")
0084 pyout.close()
0085
0086
0087 #Create crab conf
0088
0089 crab_block = """
0090 [CRAB]
0091 jobtype = cmssw
0092 scheduler = glite
0093 #server_name =
0094
0095 [EDG]
0096 remove_default_blacklist=1
0097 rb = CERN
0098
0099 [USER]
0100 return_data = 0
0101 copy_data = 1
0102 storage_element=srm-cms.cern.ch
0103 storage_path=/srm/managerv2?SFN=/castor/cern.ch/
0104 user_remote_dir=/user/n/nuno/relval/harvest/
0105 publish_data=0
0106 thresholdLevel=70
0107 eMail=nuno@cern.ch
0108
0109 [CMSSW]
0110 total_number_of_events=-1
0111 show_prod = 1
0112 """
0113
0114 crab_name="crab.cfg"
0115 os.system("touch " + crab_name)
0116 os.system("mv " + crab_name + " " + crab_name + "_old")
0117
0118 crab_cfg = open(crab_name, 'w')
0119 crab_cfg.write(crab_block)
0120
0121 rootfile = "DQM_V0001_R000000001" \
0122            + dsetpath.replace('/','__') \
0123            + ".root"
0124
0125 crab_cfg.write("number_of_jobs=" + str(njobs) + "\n")
0126 crab_cfg.write("pset=" + pyout_name + "\n")
0127 crab_cfg.write("output_file=" + rootfile + "\n")
0128 crab_cfg.write("datasetpath=" + dsetpath + "\n")
0129
0130
0131 crab_cfg.close()
0132
0133 #os.system("cat " + pyout_name)
0134 #print "Created crab conf:\t", crab_name,"\n"
0135
0136 print('\n\nCreated:\n\t %(pwd)s/%(pf)s \n\t %(pwd)s/%(cf)s' \
0137       % {'pwd' : os.environ["PWD"], 'pf' : pyout_name, 'cf' : crab_name})
0138
0139 print("Done.")