scripts/fileregistration/fileTransfer.py

0001 #!/usr/bin/env python3
0002
0003 import os, time, sys, shutil, glob, smtplib, re, subprocess
0004 import getopt as gop
0005 import zipfile as zp
0006 from datetime import datetime
0007 from email.MIMEText import MIMEText
0008
0009 SOURCEDIR = "/dqmdata/dqm/merged" #Directory where the original files are located.
0010 INJECTIONDIR = "/dqmdata/dqm/Tier0Shipping/inject"   #Directory where files get placed once they have been sent.
0011 TRASHCAN = "/dqmdata/dqm/trashcan"
0012 HOSTNAME = "srv-C2D05-19"
0013 CONFIGFILE = "/nfshome0/dqm/.transfer/myconfig.txt"
0014 INJECTIONSCRIPT = "/nfshome0/tier0/scripts/injectFileIntoTransferSystem.pl"
0015
0016 EMPTY_TRASHCAN = False
0017 WAIT_TIME = 600
0018 ################################################################################
0019 def getFileLists():
0020   newfiles={}
0021   trashfiles=[]
0022   filelist=[]
0023   for dir1, subdirs, files in os.walk(SOURCEDIR):
0024     for f in files:
0025       try:
0026         version=int(f.split("_V")[-1][:4])
0027         vlfn=f.split("_V")[0]+f.split("_V")[1][4:]
0028       except:
0029         version=1
0030         vlfn=f
0031       if vlfn in newfiles.keys() and version >= newfiles[vlfn][0]:
0032         newfiles[vlfn]=(version,"%s/%s" % (dir1,f))
0033       elif vlfn not in newfiles.keys():
0034         newfiles[vlfn]=(version,"%s/%s" % (dir1,f))
0035       else:
0036         trashfiles.append("%s/%s" % (dir1,f))
0037
0038   return (newfiles,trashfiles)
0039 #=====================================================================================
0040 def injectFile(f,renotify=False):
0041   fname=f.rsplit("/",1)[-1]
0042   dname=f.rsplit("/",1)[0]
0043   run=f.split("_R")[-1][:9]
0044   iname="%s/%s" % (INJECTIONDIR,fname)
0045   shutil.move(f,iname)
0046   parameters=["--filename %s" % fname,
0047               "--type dqm",
0048               "--path %s" % INJECTIONDIR,
0049               "--destination dqm",
0050               "--hostname %s" % HOSTNAME,
0051               "--config %s" % CONFIGFILE,
0052               "--runnumber %s" % run,
0053               "--lumisection 95",
0054               "--numevents 834474816",
0055               "--appname dqmArchive",
0056               "--appversion dqmArchive_1_0"]
0057   cmd="%s %s" % (INJECTIONSCRIPT," ".join(parameters))
0058   result = subprocess.getstatusoutput(cmd)
0059   if result[0] >= 1:
0060     output = result[1]
0061     print("Error injecting file %s to transfer system checking if it exists" % f)
0062     chkparameters=["--check","--filename %s" % fname,"--config %s" % CONFIGFILE]
0063     cmd="%s %s" % (INJECTIONSCRIPT," ".join(chkparameters))
0064     result = subprocess.getstatusoutput(cmd)
0065     if result[0]==1:
0066       if "File not found in database" in result[1]:
0067         print("Error: file %s not found in transfer database, check configuration" % f)
0068         return 0
0069       else:
0070         print("Warning: file %s already exists in transfer database" % f)
0071         return 2
0072     else:
0073       print("Error: problem checking database entry for file %s\n Error:%s" % (f,result[1]))
0074       return 0
0075   else:
0076     print("File %s injected successfully" % f)
0077   return 1
0078 #=====================================================================================
0079 def transferFiles():
0080   while True:
0081     #look for NEW files in SOURCEDIR and files that need to be cleared.
0082     newfiles,trashfiles=getFileLists()
0083
0084     #Making sure destination directories exist
0085     if not os.path.exists(TRASHCAN):
0086       os.makedirs(TRASHCAN)
0087     if not os.path.exists(INJECTIONDIR):
0088       os.makedirs(INJECTIONDIR)
0089
0090     #Dealing with trash can
0091     for tf in trashfiles:
0092       if EMPTY_TRASHCAN:
0093         os.remove(tf)
0094       else:
0095         tfname="%s/%s" % (TRASHCAN,tf.rsplit("/",1)[-1])
0096         shutil.move(tf,tfname)
0097
0098     #Down to bussines
0099     for ver,f in newfiles.values():
0100       ifr=injectFile(f)
0101     time.sleep(WAIT_TIMEt   )
0102 #=====================================================================================
0103 if __name__ == "__main__":
0104   transferFiles()