File indexing completed on 2023-03-17 11:09:55
0001
0002
0003 from __future__ import print_function
0004 import sys
0005 import os
0006
0007
0008
0009 if len(sys.argv) != 2 :
0010 print("""
0011 Usage: create_harvesting_py.py <dataset>
0012 example:
0013 create_harvesting_py.py \
0014 /RelValTTbar/CMSSW_3_1_0_pre4_STARTUP_30X_v1/GEN-SIM-RECO
0015 """)
0016 sys.exit(10)
0017
0018
0019 if os.getenv('DBSCMD_HOME','NOTSET') == 'NOTSET' :
0020 print("dbs not set!")
0021 sys.exit(11)
0022
0023 if os.getenv('CMSSW_VERSION','NOTSET') == 'NOTSET' :
0024 print("""
0025 cmssw not set!
0026 example:
0027 cmsrel CMSSW_3_1_0_pre4
0028 cd CMSSW_3_1_0_pre4/src
0029 eval `scramv1 runtime -sh`
0030 cd -
0031 """)
0032 sys.exit(12)
0033
0034 dsetpath = sys.argv[1]
0035
0036 from DBSAPI.dbsApi import DbsApi
0037 from DBSAPI.dbsException import *
0038 from DBSAPI.dbsApiException import *
0039 from DBSAPI.dbsOptions import DbsOptionParser
0040
0041 optManager = DbsOptionParser()
0042 (opts,args) = optManager.getOpt()
0043 api = DbsApi(opts.__dict__)
0044
0045 print("dataset: ", dsetpath)
0046 print("data files: ")
0047 for afile in api.listFiles(path=dsetpath):
0048 print(" %s" % afile['LogicalFileName'])
0049
0050
0051 totnevts=0
0052 for afile in api.listFiles(path=dsetpath):
0053 totnevts += afile['NumberOfEvents']
0054 njobs = 1
0055 nevtref = 9000
0056 if totnevts > nevtref : njobs = (int) (totnevts / 9000)
0057 print("Total # events: ", totnevts, \
0058 " to be executed in ", njobs, "processes")
0059
0060
0061
0062 raw_cmsdriver = "cmsDriver.py harvest -s HARVESTING:validationHarvesting --mc --conditions FrontierConditions_GlobalTag,STARTUP_30X::All --harvesting AtJobEnd --no_exec -n -1"
0063
0064 print("executing cmsdriver command:\n\t", raw_cmsdriver)
0065
0066 os.system( '`' + raw_cmsdriver + '`' )
0067
0068
0069
0070 fin_name="harvest_HARVESTING_STARTUP.py"
0071 pyout_name = "harvest.py"
0072 os.system("touch " + fin_name)
0073 os.system('mv ' + fin_name + " " + pyout_name )
0074 pyout = open(pyout_name, 'a')
0075
0076
0077 pyout.write("\n\n##additions to cmsDriver output \n")
0078 pyout.write("process.dqmSaver.workflow = '" + dsetpath + "'\n")
0079 pyout.write("process.source.fileNames = cms.untracked.vstring(\n")
0080
0081 for afile in api.listFiles(path=dsetpath):
0082 pyout.write(" '%s',\n" % afile['LogicalFileName'])
0083
0084 pyout.write(")")
0085 pyout.close()
0086
0087
0088
0089
0090 crab_block = """
0091 [CRAB]
0092 jobtype = cmssw
0093 scheduler = glite
0094 #server_name =
0095
0096 [EDG]
0097 remove_default_blacklist=1
0098 rb = CERN
0099
0100 [USER]
0101 return_data = 0
0102 copy_data = 1
0103 storage_element=srm-cms.cern.ch
0104 storage_path=/srm/managerv2?SFN=/castor/cern.ch/
0105 user_remote_dir=/user/n/nuno/relval/harvest/
0106 publish_data=0
0107 thresholdLevel=70
0108 eMail=nuno@cern.ch
0109
0110 [CMSSW]
0111 total_number_of_events=-1
0112 show_prod = 1
0113 """
0114
0115 crab_name="crab.cfg"
0116 os.system("touch " + crab_name)
0117 os.system("mv " + crab_name + " " + crab_name + "_old")
0118
0119 crab_cfg = open(crab_name, 'w')
0120 crab_cfg.write(crab_block)
0121
0122 rootfile = "DQM_V0001_R000000001" \
0123 + dsetpath.replace('/','__') \
0124 + ".root"
0125
0126 crab_cfg.write("number_of_jobs=" + str(njobs) + "\n")
0127 crab_cfg.write("pset=" + pyout_name + "\n")
0128 crab_cfg.write("output_file=" + rootfile + "\n")
0129 crab_cfg.write("datasetpath=" + dsetpath + "\n")
0130
0131
0132 crab_cfg.close()
0133
0134
0135
0136
0137 print('\n\nCreated:\n\t %(pwd)s/%(pf)s \n\t %(pwd)s/%(cf)s' \
0138 % {'pwd' : os.environ["PWD"], 'pf' : pyout_name, 'cf' : crab_name})
0139
0140 print("Done.")