scripts/filecollector/producerFileCleanner.py

0001 #!/usr/bin/env python3
0002 import os, time, sys, glob, re, smtplib, socket
0003 from email.MIMEText import MIMEText
0004 from traceback import print_exc, format_exc
0005 from datetime import datetime
0006 from subprocess import Popen,PIPE
0007 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
0008
0009 EMAIL = sys.argv[1]
0010 TFILEDONEDIR = sys.argv[2]
0011 COLLECTDIR = sys.argv[3]
0012 ORIGINALDONEDIR =sys.argv[4]
0013
0014 #Constans
0015 PRODUCER_DU_TOP= 90.0  #0% a 100%
0016 PRODUCER_DU_BOT= 50.0  #0% a 100%
0017 WAITTIME = 3600 * 4
0018 EMAILINTERVAL = 15 * 60 # Time between sent emails
0019 SENDMAIL = "/usr/sbin/sendmail" # sendmail location
0020 HOSTNAME = socket.gethostname().lower()
0021 EXEDIR = os.path.dirname(__file__)
0022 STOP_FILE = "%s/.stop" % EXEDIR
0023
0024 # Control variables
0025 lastEmailSent = 0
0026
0027 # --------------------------------------------------------------------
0028 def logme(msg, *args):
0029   procid = "[%s/%d]" % (__file__.rsplit("/", 1)[-1], os.getpid())
0030   print(datetime.now(), procid, msg % args)
0031
0032 def getDiskUsage(path):
0033   fsStats=os.statvfs(path)
0034   size=fsStats.f_bsize*fsStats.f_blocks
0035   available=fsStats.f_bavail*fsStats.f_bsize
0036   used=size-available
0037   usedPer=float(used)/size
0038   return (size,available,used,usedPer)
0039
0040 def getDirSize(path):
0041   import stat
0042   size=os.stat(path).st_blksize
0043   for directory,subdirs,files in os.walk(path):
0044     dStats=os.lstat(directory)
0045     size+=(dStats[stat.ST_NLINK]-1)*dStats[stat.ST_SIZE]
0046     for f in files:
0047       fStats=os.lstat("%s/%s" % (directory,f))
0048       fSize=fStats[stat.ST_SIZE]
0049       size+=fSize
0050
0051   return size
0052
0053 def sendmail(body="Hello from producerFileCleanner",subject= "Hello!"):
0054   scall = Popen("%s -t" % SENDMAIL, shell=True, stdin=PIPE)
0055   scall.stdin.write("To: %s\n" % EMAIL)
0056   scall.stdin.write("Subject: producerFileCleaner problem on server %s\n" %
0057                      HOSTNAME)
0058   scall.stdin.write("\n") # blank line separating headers from body
0059   scall.stdin.write("%s\n" % body)
0060   scall.stdin.close()
0061   rc = scall.wait()
0062   if rc != 0:
0063      logme("ERROR: Sendmail exit with status %s", rc)
0064
0065 # --------------------------------------------------------------------
0066 while True:
0067   #Check if you need to stop.
0068   if os.path.exists(STOP_FILE):
0069     logme("INFO: Stop file found, quitting")
0070     sys.exit(0)
0071
0072   try:
0073     try:
0074       doneSize=getDirSize(TFILEDONEDIR)
0075       diskSize,userAvailable,diskUsed,diskPUsage=getDiskUsage(TFILEDONEDIR)
0076
0077     except:
0078       doneSize=0
0079       diskSize,userAvailable,diskUsed,diskPUsage=getDiskUsage("/home")
0080
0081     diskPUsage*=100
0082     if diskPUsage < PRODUCER_DU_TOP:
0083       time.sleep(WAITTIME)
0084       continue
0085
0086     quota=int(diskSize*PRODUCER_DU_BOT/100)
0087     delQuota=diskUsed-quota
0088     if delQuota > doneSize:
0089       now = time.time()
0090       if now - EMAILINTERVAL > lastEmailSent:
0091         msg="ERROR: Something is filling up the disks, %s does not" \
0092           " have enough files to get to the Bottom Boundary of" \
0093           " %.2f%%" % (TFILEDONEDIR,PRODUCER_DU_BOT)
0094         sendmail(msg)
0095         lastEmailSent = now
0096
0097       logme("ERROR: Something is filling up the disks, %s does not" \
0098           " have enough files to get to the Bottom Boundary of" \
0099           " %.2f%%", TFILEDONEDIR, PRODUCER_DU_BOT)
0100
0101     aDelQuota=0
0102     FILE_LIST=[]
0103     for directory,subdirs,files in os.walk(TFILEDONEDIR):
0104       subdirs.sort()
0105       for f in sorted(files,key=lambda a: a[a.rfind("_R",1)+2:a.rfind("_R",1)+11]):
0106         fMatch=re.match(r"(DQM|Playback|Playback_full)_V[0-9]{4}_([0-9a-zA-Z]+)_R([0-9]{9})(_T[0-9]{8}|)\.root",f)
0107         if fMatch:
0108           subSystem=fMatch.group(2)
0109           run=fMatch.group(3)
0110           destDir="%s/%sxxxx/%sxx/DQM_V0001_%s_R%s.root" % (ORIGINALDONEDIR,run[0:5],run[0:7],subSystem,run)
0111           fullFName="%s/%s" % (directory,f)
0112           if os.stat(fullFName).st_size+aDelQuota > delQuota:
0113             break
0114
0115           FILE_LIST.append(fullFName)
0116           aDelQuota+=os.stat(fullFName).st_size
0117           if not os.path.exists(destDir):
0118             logme("WARNING: No subsystem file in repository %s for"
0119                   " file %s, deleting any way" %
0120                   (ORIGINALDONEDIR, fullFName))
0121
0122     if len(FILE_LIST):
0123       logme("INFO: Found %d files to be deleted", len(FILE_LIST))
0124
0125     #Cleanning ouput directory
0126     for directory,subdirs,files in os.walk(COLLECTDIR):
0127       #no subdiretories allowed in COLLECTDIR the  directory
0128       if subdirs:
0129         logme("ERROR: Output directory %s, must not contain"
0130               " subdirectories, cleanning", COLLECTDIR)
0131
0132       for sd in subdirs:
0133         fullSdName="%s/%s" % (directory,sd)
0134         for sdRoot,sdDirs,sdFiles in os.walk(fullSdName,topdown=False):
0135           for f in sdFiles:
0136             try:
0137               os.remove(f)
0138               logme("INFO: File %s has been removed", f)
0139             except Exception as e:
0140               logme("ERROR: Problem deleting file: [Errno %d] %s, '%s'",
0141                       e.errno, e.strerror, e.filename)
0142
0143           try:
0144             os.removedir(sdRoot)
0145             logme("INFO: File %s has been removed" , sdRoot)
0146           except Exception as e:
0147             logme("ERROR: Problem deleting directory: [Errno %d] %s, '%s'",
0148                       e.errno, e.strerror, e.filename)
0149
0150       for f in files:
0151         if re.match(r"(DQM|Playback|Playback_full)_V[0-9]{4}_([a-zA-Z]+)_R([0-9]{9})_T[0-9]{8}\.root", f):
0152           continue
0153
0154         if re.match(r".*\.tmp",f):
0155           continue
0156
0157         fullFName="%s/%s" % (directory, f)
0158         FILE_LIST.append(fullFName)
0159
0160       #cleaning tmp files:
0161       TMP_LIST=glob.glob("%s/*.tmp" % COLLECTDIR)
0162       TMP_LIST.sort(reverse=True,key=lambda x: os.stat(x).st_mtime)
0163       len(TMP_LIST) > 0 and TMP_LIST.pop(0)
0164       FILE_LIST.extend(TMP_LIST)
0165
0166     #remove files
0167     DIR_LIST=[]
0168     for f in FILE_LIST:
0169       try:
0170         os.remove(f)
0171         logme("INFO: File %s has been removed", f)
0172       except Exception as e:
0173         logme("ERROR: Problem deleting file: [Errno %d] %s, '%s'",
0174                 e.errno, e.strerror, e.filename)
0175       if os.path.dirname(f) not in DIR_LIST and COLLECTDIR not in os.path.dirname(f):
0176         DIR_LIST.append(os.path.dirname(f))
0177
0178     #remove emprty directories
0179     for d in DIR_LIST:
0180       try:
0181         os.removedirs(d)
0182         logme("INFO: Directory %s has been removed", d)
0183       except Exception as e:
0184         logme("ERROR: Directory delition failed: [Errno %d] %s, '%s'",
0185                 e.errno, e.strerror, e.filename)
0186
0187   except KeyboardInterrupt as e:
0188     sys.exit(0)
0189
0190   except Exception as e:
0191     logme('ERROR: %s', e)
0192     sendmail ('ERROR: %s\n%s' % (e, format_exc()))
0193     now = time.time()
0194     if now - EMAILINTERVAL > lastEmailSent:
0195       sendmail ('ERROR: %s\n%s' % (e, format_exc()))
0196       lastEmailSent = now
0197
0198     print_exc()
0199
0200   time.sleep(WAITTIME)
0201
0202
0203
0204