Warning, /DQM/Integration/scripts/filecollector/createOnlineInfo is written in an unsupported language. File is not indexed.
0001 #!/usr/bin/env python3
0002 import os, re, hashlib, time, sys
0003 from traceback import print_exc
0004 from datetime import datetime
0005 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
0006
0007 # Command line arguments
0008 BASE_DIR = sys.argv[1] #"/dqmdata/offline/repository/data/OnlineData"
0009 NEXT = sys.argv[2:] # Directories for the next agents in chain.
0010
0011 # Constants
0012 WAITTIME = 4 * 3600
0013 FILE_PAT=re.compile(r"DQM_V(?P<version>[0-9]{4})_(?P<subSys>[a-zA-Z0-9]+)_R(?P<runnr>[0-9]{9}).root$")
0014 FILE_M_PAT=re.compile(r"DQM_V(?P<version>[0-9]{4})_R(?P<runnr>[0-9]{9}).root$")
0015
0016 # --------------------------------------------------------------------
0017 def logme(msg, *args):
0018 procid = "[%s/%d]" % (__file__.rsplit("/", 1)[-1], os.getpid())
0019 print datetime.now(), procid, msg % args
0020
0021 def writeInfoFile(fileName, infoStr):
0022 logme("INFO: Creating File %s",fileName)
0023 infoFile=open(fileName , "w")
0024 infoFile.write("%s\n" % infoStr)
0025 infoFile.close()
0026
0027 # --------------------------------------------------------------------
0028 # Process files forever.while True:
0029 while True:
0030 try:
0031 logme("INFO: Entire Base Directory Sweep")
0032 new = []
0033 for cDir,sDirs,files in os.walk(BASE_DIR):
0034 for f in files:
0035 fMatch=FILE_PAT.match(f)
0036 fMMatch=FILE_M_PAT.match(f)
0037 fileName = "%s/%s" % (cDir,f)
0038 dqminfo = "%s.dqminfo" % fileName
0039 if not os.path.exists(dqminfo):
0040 if fMatch:
0041 subSystem = fMatch.group("subSys")
0042 runNr = int(fMatch.group("runnr"))
0043 version = int(fMatch.group("version"))
0044
0045 if fMMatch:
0046 subSystem = "Merged File"
0047 runNr = int(fMMatch.group("runnr"))
0048 version = int(fMMatch.group("version"))
0049
0050 if not fMatch and not fMMatch:
0051 continue
0052
0053 fDict={'subsystem': subSystem,
0054 'origin': '/dqmdata/dqm/uploads/%s.origin' % f,
0055 'zippat': 'OnlineData/original/%05dxxxx/DQM_Online_R%07dxx_S%%04d.zip' % (runNr/10000,runNr/100),
0056 'dataset': '/Global/Online/ALL',
0057 'import': '/dqmdata/dqm/uploads/%s' %f,
0058 'path': 'OnlineData/original/%05dxxxx/%07dxx/%s' %(runNr/10000,runNr/100,f),
0059 'check': 'VERIFY: Good to go',
0060 'size': os.stat(fileName).st_size,
0061 'runnr': runNr,
0062 'filepat': 'OnlineData/original/%05dxxxx/%07dxx/DQM_V%%04d_%s_R%09d.root' % (runNr/10000,runNr/100,subSystem,runNr),
0063 'md5sum': '%s' % hashlib.md5(file(fileName).read()).hexdigest(),
0064 'class': 'online_data',
0065 'version': version,
0066 'time': int(os.stat(fileName).st_mtime),
0067 'xpath': '/home/dqmprolocal/output/DQM_V%04d_%s_R%09d_Txxxxxxxx.root'}
0068 writeInfoFile(dqminfo, str(fDict))
0069 new.append(dqminfo)
0070
0071 info = eval(file(dqminfo).read())
0072 if 'zippath' not in info:
0073 for n in NEXT:
0074 if not os.path.exists(n):
0075 os.makedirs(n)
0076 ninfo = "%s/%s" % (n, dqminfo.rsplit("/", 1)[-1])
0077 if not os.path.exists(ninfo):
0078 os.link(dqminfo, ninfo)
0079
0080 # If anything bad happened, barf but keep going.
0081 except KeyboardInterrupt, e:
0082 sys.exit(0)
0083
0084 except Exception, e:
0085 logme('error: %s', e)
0086 print_exc()
0087
0088 time.sleep(WAITTIME)