File indexing completed on 2024-11-26 02:34:08
0001
0002
0003 import os, time, sys, shutil, glob, smtplib, re, subprocess
0004 import getopt as gop
0005 import zipfile as zp
0006 from datetime import datetime
0007 from email.MIMEText import MIMEText
0008
0009 SOURCEDIR = "/dqmdata/dqm/merged"
0010 INJECTIONDIR = "/dqmdata/dqm/Tier0Shipping/inject"
0011 TRASHCAN = "/dqmdata/dqm/trashcan"
0012 HOSTNAME = "srv-C2D05-19"
0013 CONFIGFILE = "/nfshome0/dqm/.transfer/myconfig.txt"
0014 INJECTIONSCRIPT = "/nfshome0/tier0/scripts/injectFileIntoTransferSystem.pl"
0015
0016 EMPTY_TRASHCAN = False
0017 WAIT_TIME = 600
0018
0019 def getFileLists():
0020 newfiles={}
0021 trashfiles=[]
0022 filelist=[]
0023 for dir1, subdirs, files in os.walk(SOURCEDIR):
0024 for f in files:
0025 try:
0026 version=int(f.split("_V")[-1][:4])
0027 vlfn=f.split("_V")[0]+f.split("_V")[1][4:]
0028 except:
0029 version=1
0030 vlfn=f
0031 if vlfn in newfiles.keys() and version >= newfiles[vlfn][0]:
0032 newfiles[vlfn]=(version,"%s/%s" % (dir1,f))
0033 elif vlfn not in newfiles.keys():
0034 newfiles[vlfn]=(version,"%s/%s" % (dir1,f))
0035 else:
0036 trashfiles.append("%s/%s" % (dir1,f))
0037
0038 return (newfiles,trashfiles)
0039
0040 def injectFile(f,renotify=False):
0041 fname=f.rsplit("/",1)[-1]
0042 dname=f.rsplit("/",1)[0]
0043 run=f.split("_R")[-1][:9]
0044 iname="%s/%s" % (INJECTIONDIR,fname)
0045 shutil.move(f,iname)
0046 parameters=["--filename %s" % fname,
0047 "--type dqm",
0048 "--path %s" % INJECTIONDIR,
0049 "--destination dqm",
0050 "--hostname %s" % HOSTNAME,
0051 "--config %s" % CONFIGFILE,
0052 "--runnumber %s" % run,
0053 "--lumisection 95",
0054 "--numevents 834474816",
0055 "--appname dqmArchive",
0056 "--appversion dqmArchive_1_0"]
0057 cmd="%s %s" % (INJECTIONSCRIPT," ".join(parameters))
0058 result = subprocess.getstatusoutput(cmd)
0059 if result[0] >= 1:
0060 output = result[1]
0061 print("Error injecting file %s to transfer system checking if it exists" % f)
0062 chkparameters=["--check","--filename %s" % fname,"--config %s" % CONFIGFILE]
0063 cmd="%s %s" % (INJECTIONSCRIPT," ".join(chkparameters))
0064 result = subprocess.getstatusoutput(cmd)
0065 if result[0]==1:
0066 if "File not found in database" in result[1]:
0067 print("Error: file %s not found in transfer database, check configuration" % f)
0068 return 0
0069 else:
0070 print("Warning: file %s already exists in transfer database" % f)
0071 return 2
0072 else:
0073 print("Error: problem checking database entry for file %s\n Error:%s" % (f,result[1]))
0074 return 0
0075 else:
0076 print("File %s injected successfully" % f)
0077 return 1
0078
0079 def transferFiles():
0080 while True:
0081
0082 newfiles,trashfiles=getFileLists()
0083
0084
0085 if not os.path.exists(TRASHCAN):
0086 os.makedirs(TRASHCAN)
0087 if not os.path.exists(INJECTIONDIR):
0088 os.makedirs(INJECTIONDIR)
0089
0090
0091 for tf in trashfiles:
0092 if EMPTY_TRASHCAN:
0093 os.remove(tf)
0094 else:
0095 tfname="%s/%s" % (TRASHCAN,tf.rsplit("/",1)[-1])
0096 shutil.move(tf,tfname)
0097
0098
0099 for ver,f in newfiles.values():
0100 ifr=injectFile(f)
0101 time.sleep(WAIT_TIMEt )
0102
0103 if __name__ == "__main__":
0104 transferFiles()