Common/relval/doHarvest.py

0001 #!/usr/bin/env python
0002
0003 from __future__ import print_function
0004 import sys
0005 import os
0006
0007 #Check arg,settings
0008
0009 if len(sys.argv) != 2 :
0010     print("""
0011     Usage: create_harvesting_py.py  <dataset>
0012     example:
0013     create_harvesting_py.py \
0014      /RelValTTbar/CMSSW_3_1_0_pre4_STARTUP_30X_v1/GEN-SIM-RECO
0015     """)
0016     sys.exit(10)
0017
0018 #Get data files of dataset to be processed
0019 if os.getenv('DBSCMD_HOME','NOTSET') == 'NOTSET' :
0020     print("dbs not set!")
0021     sys.exit(11)
0022
0023 if os.getenv('CMSSW_VERSION','NOTSET') == 'NOTSET' :
0024     print("""
0025     cmssw not set!
0026     example:
0027       cmsrel CMSSW_3_1_0_pre4
0028       cd CMSSW_3_1_0_pre4/src
0029       eval `scramv1 runtime -sh`
0030       cd -
0031     """)
0032     sys.exit(12)
0033
0034 dsetpath = sys.argv[1]
0035
0036 from DBSAPI.dbsApi import DbsApi
0037 from DBSAPI.dbsException import *
0038 from DBSAPI.dbsApiException import *
0039 from DBSAPI.dbsOptions import DbsOptionParser
0040
0041 optManager  = DbsOptionParser()
0042 (opts,args) = optManager.getOpt()
0043 api = DbsApi(opts.__dict__)
0044
0045 print("dataset: ", dsetpath)
0046 print("data files: ")
0047 for afile in api.listFiles(path=dsetpath):
0048   print("  %s" % afile['LogicalFileName'])
0049
0050 #Determine number of events/processes
0051 totnevts=0
0052 for afile in api.listFiles(path=dsetpath):
0053   totnevts += afile['NumberOfEvents']
0054 njobs = 1
0055 nevtref = 9000
0056 if totnevts > nevtref : njobs = (int) (totnevts / 9000)
0057 print("Total # events: ", totnevts, \
0058       " to be executed in ", njobs, "processes")
0059
0060
0061 #Run cmsDriver command
0062 raw_cmsdriver = "cmsDriver.py harvest -s HARVESTING:validationHarvesting --mc  --conditions FrontierConditions_GlobalTag,STARTUP_30X::All --harvesting AtJobEnd --no_exec -n -1"
0063
0064 print("executing cmsdriver command:\n\t", raw_cmsdriver)
0065
0066 os.system( '`' + raw_cmsdriver + '`' )
0067
0068
0069 #Open output py
0070 fin_name="harvest_HARVESTING_STARTUP.py"
0071 pyout_name = "harvest.py"
0072 os.system("touch " + fin_name)
0073 os.system('mv ' + fin_name + " " + pyout_name )
0074 pyout = open(pyout_name, 'a')
0075
0076 #Added to py config: input, output file name, dqm settings
0077 pyout.write("\n\n##additions to cmsDriver output \n")
0078 pyout.write("process.dqmSaver.workflow = '" + dsetpath + "'\n")
0079 pyout.write("process.source.fileNames = cms.untracked.vstring(\n")
0080
0081 for afile in api.listFiles(path=dsetpath):
0082     pyout.write("  '%s',\n" % afile['LogicalFileName'])
0083
0084 pyout.write(")")
0085 pyout.close()
0086
0087
0088 #Create crab conf
0089
0090 crab_block = """
0091 [CRAB]
0092 jobtype = cmssw
0093 scheduler = glite
0094 #server_name =
0095
0096 [EDG]
0097 remove_default_blacklist=1
0098 rb = CERN
0099
0100 [USER]
0101 return_data = 0
0102 copy_data = 1
0103 storage_element=srm-cms.cern.ch
0104 storage_path=/srm/managerv2?SFN=/castor/cern.ch/
0105 user_remote_dir=/user/n/nuno/relval/harvest/
0106 publish_data=0
0107 thresholdLevel=70
0108 eMail=nuno@cern.ch
0109
0110 [CMSSW]
0111 total_number_of_events=-1
0112 show_prod = 1
0113 """
0114
0115 crab_name="crab.cfg"
0116 os.system("touch " + crab_name)
0117 os.system("mv " + crab_name + " " + crab_name + "_old")
0118
0119 crab_cfg = open(crab_name, 'w')
0120 crab_cfg.write(crab_block)
0121
0122 rootfile = "DQM_V0001_R000000001" \
0123            + dsetpath.replace('/','__') \
0124            + ".root"
0125
0126 crab_cfg.write("number_of_jobs=" + str(njobs) + "\n")
0127 crab_cfg.write("pset=" + pyout_name + "\n")
0128 crab_cfg.write("output_file=" + rootfile + "\n")
0129 crab_cfg.write("datasetpath=" + dsetpath + "\n")
0130
0131
0132 crab_cfg.close()
0133
0134 #os.system("cat " + pyout_name)
0135 #print "Created crab conf:\t", crab_name,"\n"
0136
0137 print('\n\nCreated:\n\t %(pwd)s/%(pf)s \n\t %(pwd)s/%(cf)s' \
0138       % {'pwd' : os.environ["PWD"], 'pf' : pyout_name, 'cf' : crab_name})
0139
0140 print("Done.")