File indexing completed on 2023-03-17 10:55:09
0001
0002
0003 from __future__ import print_function
0004 import os, time, sys, shutil, glob, smtplib, re, subprocess
0005 import getopt as gop
0006 import zipfile as zp
0007 from datetime import datetime
0008 from email.MIMEText import MIMEText
0009
0010 SOURCEDIR = "/dqmdata/dqm/merged"
0011 INJECTIONDIR = "/dqmdata/dqm/Tier0Shipping/inject"
0012 TRASHCAN = "/dqmdata/dqm/trashcan"
0013 HOSTNAME = "srv-C2D05-19"
0014 CONFIGFILE = "/nfshome0/dqm/.transfer/myconfig.txt"
0015 INJECTIONSCRIPT = "/nfshome0/tier0/scripts/injectFileIntoTransferSystem.pl"
0016
0017 EMPTY_TRASHCAN = False
0018 WAIT_TIME = 600
0019
0020 def getFileLists():
0021 newfiles={}
0022 trashfiles=[]
0023 filelist=[]
0024 for dir1, subdirs, files in os.walk(SOURCEDIR):
0025 for f in files:
0026 try:
0027 version=int(f.split("_V")[-1][:4])
0028 vlfn=f.split("_V")[0]+f.split("_V")[1][4:]
0029 except:
0030 version=1
0031 vlfn=f
0032 if vlfn in newfiles.keys() and version >= newfiles[vlfn][0]:
0033 newfiles[vlfn]=(version,"%s/%s" % (dir1,f))
0034 elif vlfn not in newfiles.keys():
0035 newfiles[vlfn]=(version,"%s/%s" % (dir1,f))
0036 else:
0037 trashfiles.append("%s/%s" % (dir1,f))
0038
0039 return (newfiles,trashfiles)
0040
0041 def injectFile(f,renotify=False):
0042 fname=f.rsplit("/",1)[-1]
0043 dname=f.rsplit("/",1)[0]
0044 run=f.split("_R")[-1][:9]
0045 iname="%s/%s" % (INJECTIONDIR,fname)
0046 shutil.move(f,iname)
0047 parameters=["--filename %s" % fname,
0048 "--type dqm",
0049 "--path %s" % INJECTIONDIR,
0050 "--destination dqm",
0051 "--hostname %s" % HOSTNAME,
0052 "--config %s" % CONFIGFILE,
0053 "--runnumber %s" % run,
0054 "--lumisection 95",
0055 "--numevents 834474816",
0056 "--appname dqmArchive",
0057 "--appversion dqmArchive_1_0"]
0058 cmd="%s %s" % (INJECTIONSCRIPT," ".join(parameters))
0059 result = subprocess.getstatusoutput(cmd)
0060 if result[0] >= 1:
0061 output = result[1]
0062 print("Error injecting file %s to transfer system checking if it exists" % f)
0063 chkparameters=["--check","--filename %s" % fname,"--config %s" % CONFIGFILE]
0064 cmd="%s %s" % (INJECTIONSCRIPT," ".join(chkparameters))
0065 result = subprocess.getstatusoutput(cmd)
0066 if result[0]==1:
0067 if "File not found in database" in result[1]:
0068 print("Error: file %s not found in transfer database, check configuration" % f)
0069 return 0
0070 else:
0071 print("Warning: file %s already exists in transfer database" % f)
0072 return 2
0073 else:
0074 print("Error: problem checking database entry for file %s\n Error:%s" % (f,result[1]))
0075 return 0
0076 else:
0077 print("File %s injected successfully" % f)
0078 return 1
0079
0080 def transferFiles():
0081 while True:
0082
0083 newfiles,trashfiles=getFileLists()
0084
0085
0086 if not os.path.exists(TRASHCAN):
0087 os.makedirs(TRASHCAN)
0088 if not os.path.exists(INJECTIONDIR):
0089 os.makedirs(INJECTIONDIR)
0090
0091
0092 for tf in trashfiles:
0093 if EMPTY_TRASHCAN:
0094 os.remove(tf)
0095 else:
0096 tfname="%s/%s" % (TRASHCAN,tf.rsplit("/",1)[-1])
0097 shutil.move(tf,tfname)
0098
0099
0100 for ver,f in newfiles.values():
0101 ifr=injectFile(f)
0102 time.sleep(WAIT_TIMEt )
0103
0104 if __name__ == "__main__":
0105 transferFiles()