Back to home page

Project CMSSW displayed by LXR

 
 

    


Warning, /DQM/Integration/scripts/filecollector/createOnlineInfo is written in an unsupported language. File is not indexed.

0001 #!/usr/bin/env python3
0002 import os, re, hashlib, time, sys
0003 from traceback import print_exc
0004 from datetime import datetime
0005 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
0006 
0007 # Command line arguments
0008 BASE_DIR = sys.argv[1] #"/dqmdata/offline/repository/data/OnlineData"
0009 NEXT = sys.argv[2:]     # Directories for the next agents in chain.
0010 
0011 # Constants
0012 WAITTIME = 4 * 3600
0013 FILE_PAT=re.compile(r"DQM_V(?P<version>[0-9]{4})_(?P<subSys>[a-zA-Z0-9]+)_R(?P<runnr>[0-9]{9}).root$")
0014 FILE_M_PAT=re.compile(r"DQM_V(?P<version>[0-9]{4})_R(?P<runnr>[0-9]{9}).root$")
0015 
0016 # --------------------------------------------------------------------
0017 def logme(msg, *args):
0018   procid = "[%s/%d]" % (__file__.rsplit("/", 1)[-1], os.getpid())
0019   print datetime.now(), procid, msg % args
0020   
0021 def writeInfoFile(fileName, infoStr):
0022   logme("INFO: Creating File %s",fileName)
0023   infoFile=open(fileName , "w")
0024   infoFile.write("%s\n" % infoStr)
0025   infoFile.close()
0026 
0027 # --------------------------------------------------------------------
0028 # Process files forever.while True:
0029 while True:
0030   try:
0031     logme("INFO: Entire Base Directory Sweep")
0032     new = []
0033     for cDir,sDirs,files in os.walk(BASE_DIR):
0034       for f in files:
0035         fMatch=FILE_PAT.match(f)
0036         fMMatch=FILE_M_PAT.match(f)
0037         fileName  = "%s/%s" % (cDir,f)
0038         dqminfo = "%s.dqminfo" % fileName
0039         if not os.path.exists(dqminfo):        
0040           if fMatch:
0041             subSystem = fMatch.group("subSys")
0042             runNr     = int(fMatch.group("runnr"))
0043             version   = int(fMatch.group("version"))
0044           
0045           if fMMatch:
0046             subSystem = "Merged File"
0047             runNr     = int(fMMatch.group("runnr"))
0048             version   = int(fMMatch.group("version")) 
0049           
0050           if not fMatch and not fMMatch:
0051             continue
0052           
0053           fDict={'subsystem': subSystem,
0054                     'origin': '/dqmdata/dqm/uploads/%s.origin' % f,
0055                     'zippat': 'OnlineData/original/%05dxxxx/DQM_Online_R%07dxx_S%%04d.zip' % (runNr/10000,runNr/100), 
0056                     'dataset': '/Global/Online/ALL', 
0057                     'import': '/dqmdata/dqm/uploads/%s' %f, 
0058                     'path': 'OnlineData/original/%05dxxxx/%07dxx/%s' %(runNr/10000,runNr/100,f), 
0059                     'check': 'VERIFY: Good to go', 
0060                     'size': os.stat(fileName).st_size, 
0061                     'runnr': runNr, 
0062                     'filepat': 'OnlineData/original/%05dxxxx/%07dxx/DQM_V%%04d_%s_R%09d.root' % (runNr/10000,runNr/100,subSystem,runNr), 
0063                     'md5sum': '%s' %  hashlib.md5(file(fileName).read()).hexdigest(), 
0064                     'class': 'online_data', 
0065                     'version': version, 
0066                     'time': int(os.stat(fileName).st_mtime), 
0067                     'xpath': '/home/dqmprolocal/output/DQM_V%04d_%s_R%09d_Txxxxxxxx.root'}
0068           writeInfoFile(dqminfo, str(fDict))
0069           new.append(dqminfo)
0070           
0071         info = eval(file(dqminfo).read())
0072         if 'zippath' not in info:
0073           for n in NEXT:
0074             if not os.path.exists(n):
0075               os.makedirs(n)
0076             ninfo = "%s/%s" % (n, dqminfo.rsplit("/", 1)[-1])
0077             if not os.path.exists(ninfo):
0078               os.link(dqminfo, ninfo)
0079 
0080   # If anything bad happened, barf but keep going.
0081   except KeyboardInterrupt, e:
0082     sys.exit(0)
0083 
0084   except Exception, e:
0085     logme('error: %s', e)
0086     print_exc() 
0087 
0088   time.sleep(WAITTIME)