File indexing completed on 2024-12-01 23:40:22
0001
0002
0003 """:"
0004
0005 python_cmd="python"
0006 python3 -c "from FWCore.PythonFramework.CmsRun import CmsRun" 2>/dev/null && python_cmd="python3"
0007 exec ${python_cmd} $0 ${1+"$@"}
0008
0009 """
0010
0011 print('Starting cmsLHEtoEOSManager.py')
0012
0013 __version__ = "$Revision: 1.13 $"
0014
0015 import os
0016 import subprocess
0017 import time
0018 import re
0019
0020 defaultEOSRootPath = '/eos/cms/store/lhe'
0021 if "CMSEOS_LHE_ROOT_DIRECTORY" in os.environ:
0022 defaultEOSRootPath = os.environ["CMSEOS_LHE_ROOT_DIRECTORY"]
0023 defaultEOSLoadPath = 'root://eoscms.cern.ch/'
0024 defaultEOSlistCommand = 'xrdfs '+defaultEOSLoadPath+' ls '
0025 defaultEOSmkdirCommand = 'xrdfs '+defaultEOSLoadPath+' mkdir '
0026 defaultEOSfeCommand = 'xrdfs '+defaultEOSLoadPath+' stat -q IsReadable '
0027 defaultEOSchecksumCommand = 'xrdfs '+defaultEOSLoadPath+' query checksum '
0028 defaultEOScpCommand = 'xrdcp -np '
0029
0030 def findXrdDir(theDirRecord):
0031
0032 elements = theDirRecord.split(' ')
0033 if len(elements):
0034 return elements[-1].rstrip('\n').split('/')[-1]
0035 else:
0036 return None
0037
0038 def articleExist(artId):
0039
0040 itExists = False
0041 theCommand = defaultEOSlistCommand+' '+defaultEOSRootPath
0042 dirList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE, universal_newlines=True)
0043 for line in dirList.stdout.readlines():
0044 if findXrdDir(line) == str(artId):
0045 itExists = True
0046
0047 return itExists
0048
0049 def lastArticle():
0050
0051 artList = [0]
0052
0053 theCommand = defaultEOSlistCommand+' '+defaultEOSRootPath
0054 dirList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE, universal_newlines=True)
0055 for line in dirList.stdout.readlines():
0056 try:
0057 if line.rstrip('\n') != '':
0058 artList.append(int(findXrdDir(line)))
0059 except:
0060 break
0061
0062 return max(artList)
0063
0064
0065 def fileUpload(uploadPath,lheList, checkSumList, reallyDoIt, force=False):
0066
0067 inUploadScript = ''
0068 index = 0
0069 for f in lheList:
0070 realFileName = f.split('/')[-1]
0071
0072 newFileName = uploadPath+'/'+str(realFileName)
0073 addFile = True
0074 additionalOption = ''
0075 theCommand = defaultEOSfeCommand+' '+newFileName
0076 exeFullList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE, universal_newlines=True)
0077 result = exeFullList.stdout.readlines()
0078 if [line for line in result if ("flags:" in line.lower()) and ("isreadable" in line.lower())] and (not force):
0079 addFile = False
0080 print('File '+newFileName+' already exists: do you want to overwrite? [y/n]')
0081 reply = raw_input()
0082 if reply == 'y' or reply == 'Y':
0083 addFile = True
0084 additionalOption = ' -f '
0085 print('')
0086 print('Overwriting file '+newFileName+'\n')
0087
0088 if addFile:
0089
0090 inUploadScript = defaultEOScpCommand + additionalOption + ' ' + str(f) + ' ' + defaultEOSLoadPath+uploadPath + '/' + str(realFileName)
0091 print('Uploading file %s...' % str(f))
0092 if reallyDoIt:
0093 exeRealUpload = subprocess.Popen(["/bin/sh","-c",inUploadScript])
0094 exeRealUpload.communicate()
0095 eosCheckSumCommand = defaultEOSchecksumCommand + uploadPath + '/' + str(realFileName) + ' | awk \'{print $2}\' | cut -d= -f2'
0096 exeEosCheckSum = subprocess.Popen(eosCheckSumCommand ,shell=True, stdout=subprocess.PIPE, universal_newlines=True)
0097 EosCheckSum = exeEosCheckSum.stdout.read()
0098 assert exeEosCheckSum.wait() == 0
0099
0100 if checkSumList[index] not in EosCheckSum:
0101 print('WARNING! The checksum for file ' + str(realFileName) + ' in EOS\n')
0102 print(EosCheckSum + '\n')
0103 print('does not match the checksum of the original one\n')
0104 print(checkSumList[index] + '\n')
0105 print('please try to re-upload file ' + str(realFileName) + ' to EOS.\n')
0106 else:
0107 print('Checksum OK for file ' + str(realFileName))
0108 index = index+1
0109
0110
0111
0112
0113
0114
0115
0116 print('\n Upload ended at '+time.asctime(time.localtime(time.time())))
0117
0118
0119
0120 if __name__ == '__main__':
0121
0122 import optparse
0123
0124
0125 usage='cmsLHEtoEOSManager.py <options>'
0126 parser = optparse.OptionParser(usage)
0127 parser.add_option('-f', '--file',
0128 help='LHE local file list to be uploaded, separated by ","' ,
0129 default='',
0130 dest='fileList')
0131
0132 parser.add_option('-F', '--files-from', metavar = 'FILE',
0133 help='File containing the list of LHE local files be uploaded, one file per line')
0134
0135 parser.add_option('-n', '--new',
0136 help='Create a new article' ,
0137 action='store_true',
0138 default=False,
0139 dest='newId')
0140
0141 parser.add_option('-u', '--update',
0142 help='Update the article <Id>' ,
0143 default=0,
0144 type=int,
0145 dest='artIdUp')
0146
0147 parser.add_option('-l', '--list',
0148 help='List the files in article <Id>' ,
0149 default=0,
0150 type=int,
0151 dest='artIdLi')
0152
0153 parser.add_option('-d', '--dry-run',
0154 help='dry run, it does nothing, but you can see what it would do',
0155 action='store_true',
0156 default=False,
0157 dest='dryRun')
0158
0159 parser.add_option('-c', '--compress',
0160 help='compress the local .lhe file with xz before upload',
0161 action='store_true',
0162 default=False,
0163 dest='compress')
0164
0165 parser.add_option('--force',
0166 help='Force update if file already exists.',
0167 action='store_true',
0168 default=False,
0169 dest='force')
0170
0171 (options,args) = parser.parse_args()
0172
0173
0174
0175 print('')
0176 print('cmsLHEtoEOSmanager '+__version__[1:-1])
0177 print('')
0178 print('Running on ',time.asctime(time.localtime(time.time())))
0179 print('')
0180
0181 reallyDoIt = not options.dryRun
0182
0183
0184 if not options.newId and options.artIdUp==0 and options.artIdLi==0:
0185 raise Exception('Please specify the action to be taken, either "-n", "-u" or "-l"!')
0186
0187 if options.fileList == '' and not options.files_from and (options.newId or options.artIdUp!=0):
0188 raise Exception('Please provide the input file list!')
0189
0190 if (options.newId and (options.artIdUp != 0 or options.artIdLi != 0)) or (options.artIdUp != 0 and options.artIdLi != 0):
0191 raise Exception('Options "-n", "-u" and "-l" are mutually exclusive, please choose only one!')
0192
0193 if options.newId:
0194 print('Action: create new article\n')
0195 elif options.artIdUp != 0:
0196 print('Action: update article '+str(options.artIdUp)+'\n')
0197 elif options.artIdLi != 0:
0198 print('Action: list content of article '+str(options.artIdLi)+'\n')
0199
0200 if options.artIdLi==0:
0201 theList = []
0202 if len(options.fileList) > 0:
0203 theList=(options.fileList.split(','))
0204
0205 if options.files_from:
0206 try:
0207 f = open(options.files_from)
0208 except IOError:
0209 raise Exception('Cannot open the file list, \'%s\'' % options.files_from)
0210 for l in f:
0211 l = l.strip()
0212 if len(l) == 0 or l[0] == '#':
0213 continue
0214 theList.append(l)
0215
0216 theCompressedFilesList = []
0217 theCheckSumList = []
0218 for f in theList:
0219
0220 print(f)
0221 if not ( f.lower().endswith(".lhe") or f.lower().endswith(".lhe.xz") ):
0222 raise Exception('Input file name must have the "lhe" or "lhe.xz" final extension!')
0223 if( f.lower().endswith(".lhe.xz") ):
0224 print("Important! Input file "+f+" is already zipped: please make sure you verified its integrity with xmllint before zipping it. You can do it with:\n")
0225 print("xmllint file.lhe\n")
0226 print("Otherwise it is best to pass the unzipped file to this script and let it check its integrity and compress the file with the --compress option\n")
0227
0228 if not os.path.exists(f):
0229 raise Exception('Input file '+f+' does not exists')
0230 if( f.lower().endswith(".lhe") ):
0231 theCheckIntegrityCommand = 'xmllint -noout '+f
0232 exeCheckIntegrity = subprocess.Popen(["/bin/sh","-c", theCheckIntegrityCommand])
0233 intCode = exeCheckIntegrity.wait()
0234 if(intCode != 0):
0235 raise Exception('Input file '+f+ ' is corrupted')
0236 if reallyDoIt and options.compress:
0237 print("Compressing file",f)
0238 if( f.lower().endswith(".lhe.xz") ):
0239 raise Exception('Input file '+f+' is already compressed! This is inconsistent with the --compress option!')
0240 theCompressionCommand = 'xz '+f
0241 exeCompression = subprocess.Popen(["/bin/sh","-c",theCompressionCommand])
0242 exeCompression.communicate()
0243 theCompressedFilesList.append(f+'.xz')
0244 if reallyDoIt and options.compress:
0245 theList = theCompressedFilesList
0246 for f in theList:
0247 try:
0248 exeCheckSum = subprocess.Popen(["/afs/cern.ch/cms/caf/bin/cms_adler32",f], stdout=subprocess.PIPE, universal_newlines=True)
0249 getCheckSum = subprocess.Popen(["awk", "{print $1}"], stdin=exeCheckSum.stdout, stdout=subprocess.PIPE, universal_newlines=True)
0250 exeCheckSum.stdout.close()
0251 output,err = getCheckSum.communicate()
0252 theCheckSumList.append(output.strip())
0253 except:
0254 theCheckSumList.append("missing-adler32")
0255
0256 newArt = 0
0257 uploadPath = ''
0258
0259
0260
0261 if options.newId:
0262 oldArt = lastArticle()
0263 newArt = oldArt+1
0264 print('Creating new article with identifier '+str(newArt)+' ...\n')
0265 uploadPath = defaultEOSRootPath+'/'+str(newArt)
0266 theCommand = defaultEOSmkdirCommand+' '+uploadPath
0267 if reallyDoIt:
0268 exeUpload = subprocess.Popen(["/bin/sh","-c",theCommand])
0269 exeUpload.communicate()
0270
0271
0272
0273 elif options.artIdUp != 0:
0274 newArt = options.artIdUp
0275 if articleExist(newArt):
0276 uploadPath = defaultEOSRootPath+'/'+str(newArt)
0277 else:
0278 raise Exception('Article '+str(newArt)+' to be updated does not exist!')
0279
0280
0281
0282 elif options.artIdLi !=0:
0283 listPath = defaultEOSRootPath+'/'+str(options.artIdLi)
0284 theCommand = defaultEOSlistCommand+' '+listPath
0285 exeList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE, universal_newlines=True)
0286 for line in exeList.stdout.readlines():
0287 if findXrdDir(line) != None:
0288 print(findXrdDir(line))
0289
0290
0291 if newArt > 0:
0292 fileUpload(uploadPath,theList, theCheckSumList, reallyDoIt, options.force)
0293 listPath = defaultEOSRootPath+'/'+str(newArt)
0294 print('')
0295 print('Listing the '+str(newArt)+' article content after upload:')
0296 theCommand = defaultEOSlistCommand+' '+listPath
0297 if reallyDoIt:
0298 exeFullList = subprocess.Popen(["/bin/sh","-c",theCommand])
0299 exeFullList.communicate()
0300 else:
0301 print('Dry run, nothing was done')
0302