scripts/fileregistration/fileTransfer.py

0001 #!/usr/bin/env python3
0002
0003 from __future__ import print_function
0004 import os, time, sys, shutil, glob, smtplib, re, subprocess
0005 import getopt as gop
0006 import zipfile as zp
0007 from datetime import datetime
0008 from email.MIMEText import MIMEText
0009
0010 SOURCEDIR = "/dqmdata/dqm/merged" #Directory where the original files are located.
0011 INJECTIONDIR = "/dqmdata/dqm/Tier0Shipping/inject"   #Directory where files get placed once they have been sent.
0012 TRASHCAN = "/dqmdata/dqm/trashcan"
0013 HOSTNAME = "srv-C2D05-19"
0014 CONFIGFILE = "/nfshome0/dqm/.transfer/myconfig.txt"
0015 INJECTIONSCRIPT = "/nfshome0/tier0/scripts/injectFileIntoTransferSystem.pl"
0016
0017 EMPTY_TRASHCAN = False
0018 WAIT_TIME = 600
0019 ################################################################################
0020 def getFileLists():
0021   newfiles={}
0022   trashfiles=[]
0023   filelist=[]
0024   for dir1, subdirs, files in os.walk(SOURCEDIR):
0025     for f in files:
0026       try:
0027         version=int(f.split("_V")[-1][:4])
0028         vlfn=f.split("_V")[0]+f.split("_V")[1][4:]
0029       except:
0030         version=1
0031         vlfn=f
0032       if vlfn in newfiles.keys() and version >= newfiles[vlfn][0]:
0033         newfiles[vlfn]=(version,"%s/%s" % (dir1,f))
0034       elif vlfn not in newfiles.keys():
0035         newfiles[vlfn]=(version,"%s/%s" % (dir1,f))
0036       else:
0037         trashfiles.append("%s/%s" % (dir1,f))
0038
0039   return (newfiles,trashfiles)
0040 #=====================================================================================
0041 def injectFile(f,renotify=False):
0042   fname=f.rsplit("/",1)[-1]
0043   dname=f.rsplit("/",1)[0]
0044   run=f.split("_R")[-1][:9]
0045   iname="%s/%s" % (INJECTIONDIR,fname)
0046   shutil.move(f,iname)
0047   parameters=["--filename %s" % fname,
0048               "--type dqm",
0049               "--path %s" % INJECTIONDIR,
0050               "--destination dqm",
0051               "--hostname %s" % HOSTNAME,
0052               "--config %s" % CONFIGFILE,
0053               "--runnumber %s" % run,
0054               "--lumisection 95",
0055               "--numevents 834474816",
0056               "--appname dqmArchive",
0057               "--appversion dqmArchive_1_0"]
0058   cmd="%s %s" % (INJECTIONSCRIPT," ".join(parameters))
0059   result = subprocess.getstatusoutput(cmd)
0060   if result[0] >= 1:
0061     output = result[1]
0062     print("Error injecting file %s to transfer system checking if it exists" % f)
0063     chkparameters=["--check","--filename %s" % fname,"--config %s" % CONFIGFILE]
0064     cmd="%s %s" % (INJECTIONSCRIPT," ".join(chkparameters))
0065     result = subprocess.getstatusoutput(cmd)
0066     if result[0]==1:
0067       if "File not found in database" in result[1]:
0068         print("Error: file %s not found in transfer database, check configuration" % f)
0069         return 0
0070       else:
0071         print("Warning: file %s already exists in transfer database" % f)
0072         return 2
0073     else:
0074       print("Error: problem checking database entry for file %s\n Error:%s" % (f,result[1]))
0075       return 0
0076   else:
0077     print("File %s injected successfully" % f)
0078   return 1
0079 #=====================================================================================
0080 def transferFiles():
0081   while True:
0082     #look for NEW files in SOURCEDIR and files that need to be cleared.
0083     newfiles,trashfiles=getFileLists()
0084
0085     #Making sure destination directories exist
0086     if not os.path.exists(TRASHCAN):
0087       os.makedirs(TRASHCAN)
0088     if not os.path.exists(INJECTIONDIR):
0089       os.makedirs(INJECTIONDIR)
0090
0091     #Dealing with trash can
0092     for tf in trashfiles:
0093       if EMPTY_TRASHCAN:
0094         os.remove(tf)
0095       else:
0096         tfname="%s/%s" % (TRASHCAN,tf.rsplit("/",1)[-1])
0097         shutil.move(tf,tfname)
0098
0099     #Down to bussines
0100     for ver,f in newfiles.values():
0101       ifr=injectFile(f)
0102     time.sleep(WAIT_TIMEt   )
0103 #=====================================================================================
0104 if __name__ == "__main__":
0105   transferFiles()