File indexing completed on 2023-03-17 11:04:47
0001
0002
0003 """:"
0004
0005 python_cmd="python"
0006 python3 -c "from FWCore.PythonFramework.CmsRun import CmsRun" 2>/dev/null && python_cmd="python3"
0007 exec ${python_cmd} $0 ${1+"$@"}
0008
0009 """
0010
0011 from __future__ import print_function
0012 print('Starting cmsLHEtoEOSManager.py')
0013
0014 __version__ = "$Revision: 1.13 $"
0015
0016 import os
0017 import subprocess
0018 import time
0019 import re
0020
0021 defaultEOSRootPath = '/eos/cms/store/lhe'
0022 if "CMSEOS_LHE_ROOT_DIRECTORY" in os.environ:
0023 defaultEOSRootPath = os.environ["CMSEOS_LHE_ROOT_DIRECTORY"]
0024 defaultEOSLoadPath = 'root://eoscms.cern.ch/'
0025 defaultEOSlistCommand = 'xrdfs '+defaultEOSLoadPath+' ls '
0026 defaultEOSmkdirCommand = 'xrdfs '+defaultEOSLoadPath+' mkdir '
0027 defaultEOSfeCommand = 'xrdfs '+defaultEOSLoadPath+' stat -q IsReadable '
0028 defaultEOSchecksumCommand = 'xrdfs '+defaultEOSLoadPath+' query checksum '
0029 defaultEOScpCommand = 'xrdcp -np '
0030
0031 def findXrdDir(theDirRecord):
0032
0033 elements = theDirRecord.split(' ')
0034 if len(elements):
0035 return elements[-1].rstrip('\n').split('/')[-1]
0036 else:
0037 return None
0038
0039 def articleExist(artId):
0040
0041 itExists = False
0042 theCommand = defaultEOSlistCommand+' '+defaultEOSRootPath
0043 dirList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE, universal_newlines=True)
0044 for line in dirList.stdout.readlines():
0045 if findXrdDir(line) == str(artId):
0046 itExists = True
0047
0048 return itExists
0049
0050 def lastArticle():
0051
0052 artList = [0]
0053
0054 theCommand = defaultEOSlistCommand+' '+defaultEOSRootPath
0055 dirList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE, universal_newlines=True)
0056 for line in dirList.stdout.readlines():
0057 try:
0058 if line.rstrip('\n') != '':
0059 artList.append(int(findXrdDir(line)))
0060 except:
0061 break
0062
0063 return max(artList)
0064
0065
0066 def fileUpload(uploadPath,lheList, checkSumList, reallyDoIt, force=False):
0067
0068 inUploadScript = ''
0069 index = 0
0070 for f in lheList:
0071 realFileName = f.split('/')[-1]
0072
0073 newFileName = uploadPath+'/'+str(realFileName)
0074 addFile = True
0075 additionalOption = ''
0076 theCommand = defaultEOSfeCommand+' '+newFileName
0077 exeFullList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE, universal_newlines=True)
0078 result = exeFullList.stdout.readlines()
0079 if [line for line in result if ("flags:" in line.lower()) and ("isreadable" in line.lower())] and (not force):
0080 addFile = False
0081 print('File '+newFileName+' already exists: do you want to overwrite? [y/n]')
0082 reply = raw_input()
0083 if reply == 'y' or reply == 'Y':
0084 addFile = True
0085 additionalOption = ' -f '
0086 print('')
0087 print('Overwriting file '+newFileName+'\n')
0088
0089 if addFile:
0090
0091 inUploadScript = defaultEOScpCommand + additionalOption + ' ' + str(f) + ' ' + defaultEOSLoadPath+uploadPath + '/' + str(realFileName)
0092 print('Uploading file %s...' % str(f))
0093 if reallyDoIt:
0094 exeRealUpload = subprocess.Popen(["/bin/sh","-c",inUploadScript])
0095 exeRealUpload.communicate()
0096 eosCheckSumCommand = defaultEOSchecksumCommand + uploadPath + '/' + str(realFileName) + ' | awk \'{print $2}\' | cut -d= -f2'
0097 exeEosCheckSum = subprocess.Popen(eosCheckSumCommand ,shell=True, stdout=subprocess.PIPE, universal_newlines=True)
0098 EosCheckSum = exeEosCheckSum.stdout.read()
0099 assert exeEosCheckSum.wait() == 0
0100
0101 if checkSumList[index] not in EosCheckSum:
0102 print('WARNING! The checksum for file ' + str(realFileName) + ' in EOS\n')
0103 print(EosCheckSum + '\n')
0104 print('does not match the checksum of the original one\n')
0105 print(checkSumList[index] + '\n')
0106 print('please try to re-upload file ' + str(realFileName) + ' to EOS.\n')
0107 else:
0108 print('Checksum OK for file ' + str(realFileName))
0109 index = index+1
0110
0111
0112
0113
0114
0115
0116
0117 print('\n Upload ended at '+time.asctime(time.localtime(time.time())))
0118
0119
0120
0121 if __name__ == '__main__':
0122
0123 import optparse
0124
0125
0126 usage='cmsLHEtoEOSManager.py <options>'
0127 parser = optparse.OptionParser(usage)
0128 parser.add_option('-f', '--file',
0129 help='LHE local file list to be uploaded, separated by ","' ,
0130 default='',
0131 dest='fileList')
0132
0133 parser.add_option('-F', '--files-from', metavar = 'FILE',
0134 help='File containing the list of LHE local files be uploaded, one file per line')
0135
0136 parser.add_option('-n', '--new',
0137 help='Create a new article' ,
0138 action='store_true',
0139 default=False,
0140 dest='newId')
0141
0142 parser.add_option('-u', '--update',
0143 help='Update the article <Id>' ,
0144 default=0,
0145 type=int,
0146 dest='artIdUp')
0147
0148 parser.add_option('-l', '--list',
0149 help='List the files in article <Id>' ,
0150 default=0,
0151 type=int,
0152 dest='artIdLi')
0153
0154 parser.add_option('-d', '--dry-run',
0155 help='dry run, it does nothing, but you can see what it would do',
0156 action='store_true',
0157 default=False,
0158 dest='dryRun')
0159
0160 parser.add_option('-c', '--compress',
0161 help='compress the local .lhe file with xz before upload',
0162 action='store_true',
0163 default=False,
0164 dest='compress')
0165
0166 parser.add_option('--force',
0167 help='Force update if file already exists.',
0168 action='store_true',
0169 default=False,
0170 dest='force')
0171
0172 (options,args) = parser.parse_args()
0173
0174
0175
0176 print('')
0177 print('cmsLHEtoEOSmanager '+__version__[1:-1])
0178 print('')
0179 print('Running on ',time.asctime(time.localtime(time.time())))
0180 print('')
0181
0182 reallyDoIt = not options.dryRun
0183
0184
0185 if not options.newId and options.artIdUp==0 and options.artIdLi==0:
0186 raise Exception('Please specify the action to be taken, either "-n", "-u" or "-l"!')
0187
0188 if options.fileList == '' and not options.files_from and (options.newId or options.artIdUp!=0):
0189 raise Exception('Please provide the input file list!')
0190
0191 if (options.newId and (options.artIdUp != 0 or options.artIdLi != 0)) or (options.artIdUp != 0 and options.artIdLi != 0):
0192 raise Exception('Options "-n", "-u" and "-l" are mutually exclusive, please choose only one!')
0193
0194 if options.newId:
0195 print('Action: create new article\n')
0196 elif options.artIdUp != 0:
0197 print('Action: update article '+str(options.artIdUp)+'\n')
0198 elif options.artIdLi != 0:
0199 print('Action: list content of article '+str(options.artIdLi)+'\n')
0200
0201 if options.artIdLi==0:
0202 theList = []
0203 if len(options.fileList) > 0:
0204 theList=(options.fileList.split(','))
0205
0206 if options.files_from:
0207 try:
0208 f = open(options.files_from)
0209 except IOError:
0210 raise Exception('Cannot open the file list, \'%s\'' % options.files_from)
0211 for l in f:
0212 l = l.strip()
0213 if len(l) == 0 or l[0] == '#':
0214 continue
0215 theList.append(l)
0216
0217 theCompressedFilesList = []
0218 theCheckSumList = []
0219 for f in theList:
0220
0221 print(f)
0222 if not ( f.lower().endswith(".lhe") or f.lower().endswith(".lhe.xz") ):
0223 raise Exception('Input file name must have the "lhe" or "lhe.xz" final extension!')
0224 if( f.lower().endswith(".lhe.xz") ):
0225 print("Important! Input file "+f+" is already zipped: please make sure you verified its integrity with xmllint before zipping it. You can do it with:\n")
0226 print("xmllint file.lhe\n")
0227 print("Otherwise it is best to pass the unzipped file to this script and let it check its integrity and compress the file with the --compress option\n")
0228
0229 if not os.path.exists(f):
0230 raise Exception('Input file '+f+' does not exists')
0231 if( f.lower().endswith(".lhe") ):
0232 theCheckIntegrityCommand = 'xmllint -noout '+f
0233 exeCheckIntegrity = subprocess.Popen(["/bin/sh","-c", theCheckIntegrityCommand])
0234 intCode = exeCheckIntegrity.wait()
0235 if(intCode != 0):
0236 raise Exception('Input file '+f+ ' is corrupted')
0237 if reallyDoIt and options.compress:
0238 print("Compressing file",f)
0239 if( f.lower().endswith(".lhe.xz") ):
0240 raise Exception('Input file '+f+' is already compressed! This is inconsistent with the --compress option!')
0241 theCompressionCommand = 'xz '+f
0242 exeCompression = subprocess.Popen(["/bin/sh","-c",theCompressionCommand])
0243 exeCompression.communicate()
0244 theCompressedFilesList.append(f+'.xz')
0245 if reallyDoIt and options.compress:
0246 theList = theCompressedFilesList
0247 for f in theList:
0248 try:
0249 exeCheckSum = subprocess.Popen(["/afs/cern.ch/cms/caf/bin/cms_adler32",f], stdout=subprocess.PIPE, universal_newlines=True)
0250 getCheckSum = subprocess.Popen(["awk", "{print $1}"], stdin=exeCheckSum.stdout, stdout=subprocess.PIPE, universal_newlines=True)
0251 exeCheckSum.stdout.close()
0252 output,err = getCheckSum.communicate()
0253 theCheckSumList.append(output.strip())
0254 except:
0255 theCheckSumList.append("missing-adler32")
0256
0257 newArt = 0
0258 uploadPath = ''
0259
0260
0261
0262 if options.newId:
0263 oldArt = lastArticle()
0264 newArt = oldArt+1
0265 print('Creating new article with identifier '+str(newArt)+' ...\n')
0266 uploadPath = defaultEOSRootPath+'/'+str(newArt)
0267 theCommand = defaultEOSmkdirCommand+' '+uploadPath
0268 if reallyDoIt:
0269 exeUpload = subprocess.Popen(["/bin/sh","-c",theCommand])
0270 exeUpload.communicate()
0271
0272
0273
0274 elif options.artIdUp != 0:
0275 newArt = options.artIdUp
0276 if articleExist(newArt):
0277 uploadPath = defaultEOSRootPath+'/'+str(newArt)
0278 else:
0279 raise Exception('Article '+str(newArt)+' to be updated does not exist!')
0280
0281
0282
0283 elif options.artIdLi !=0:
0284 listPath = defaultEOSRootPath+'/'+str(options.artIdLi)
0285 theCommand = defaultEOSlistCommand+' '+listPath
0286 exeList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE, universal_newlines=True)
0287 for line in exeList.stdout.readlines():
0288 if findXrdDir(line) != None:
0289 print(findXrdDir(line))
0290
0291
0292 if newArt > 0:
0293 fileUpload(uploadPath,theList, theCheckSumList, reallyDoIt, options.force)
0294 listPath = defaultEOSRootPath+'/'+str(newArt)
0295 print('')
0296 print('Listing the '+str(newArt)+' article content after upload:')
0297 theCommand = defaultEOSlistCommand+' '+listPath
0298 if reallyDoIt:
0299 exeFullList = subprocess.Popen(["/bin/sh","-c",theCommand])
0300 exeFullList.communicate()
0301 else:
0302 print('Dry run, nothing was done')
0303