scripts/filecollector/producerFileCleanner.py

0001 #!/usr/bin/env python3
0002 from __future__ import print_function
0003 import os, time, sys, glob, re, smtplib, socket
0004 from email.MIMEText import MIMEText
0005 from traceback import print_exc, format_exc
0006 from datetime import datetime
0007 from subprocess import Popen,PIPE
0008 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
0009
0010 EMAIL = sys.argv[1]
0011 TFILEDONEDIR = sys.argv[2]
0012 COLLECTDIR = sys.argv[3]
0013 ORIGINALDONEDIR =sys.argv[4]
0014
0015 #Constans
0016 PRODUCER_DU_TOP= 90.0  #0% a 100%
0017 PRODUCER_DU_BOT= 50.0  #0% a 100%
0018 WAITTIME = 3600 * 4
0019 EMAILINTERVAL = 15 * 60 # Time between sent emails
0020 SENDMAIL = "/usr/sbin/sendmail" # sendmail location
0021 HOSTNAME = socket.gethostname().lower()
0022 EXEDIR = os.path.dirname(__file__)
0023 STOP_FILE = "%s/.stop" % EXEDIR
0024
0025 # Control variables
0026 lastEmailSent = 0
0027
0028 # --------------------------------------------------------------------
0029 def logme(msg, *args):
0030   procid = "[%s/%d]" % (__file__.rsplit("/", 1)[-1], os.getpid())
0031   print(datetime.now(), procid, msg % args)
0032
0033 def getDiskUsage(path):
0034   fsStats=os.statvfs(path)
0035   size=fsStats.f_bsize*fsStats.f_blocks
0036   available=fsStats.f_bavail*fsStats.f_bsize
0037   used=size-available
0038   usedPer=float(used)/size
0039   return (size,available,used,usedPer)
0040
0041 def getDirSize(path):
0042   import stat
0043   size=os.stat(path).st_blksize
0044   for directory,subdirs,files in os.walk(path):
0045     dStats=os.lstat(directory)
0046     size+=(dStats[stat.ST_NLINK]-1)*dStats[stat.ST_SIZE]
0047     for f in files:
0048       fStats=os.lstat("%s/%s" % (directory,f))
0049       fSize=fStats[stat.ST_SIZE]
0050       size+=fSize
0051
0052   return size
0053
0054 def sendmail(body="Hello from producerFileCleanner",subject= "Hello!"):
0055   scall = Popen("%s -t" % SENDMAIL, shell=True, stdin=PIPE)
0056   scall.stdin.write("To: %s\n" % EMAIL)
0057   scall.stdin.write("Subject: producerFileCleaner problem on server %s\n" %
0058                      HOSTNAME)
0059   scall.stdin.write("\n") # blank line separating headers from body
0060   scall.stdin.write("%s\n" % body)
0061   scall.stdin.close()
0062   rc = scall.wait()
0063   if rc != 0:
0064      logme("ERROR: Sendmail exit with status %s", rc)
0065
0066 # --------------------------------------------------------------------
0067 while True:
0068   #Check if you need to stop.
0069   if os.path.exists(STOP_FILE):
0070     logme("INFO: Stop file found, quitting")
0071     sys.exit(0)
0072
0073   try:
0074     try:
0075       doneSize=getDirSize(TFILEDONEDIR)
0076       diskSize,userAvailable,diskUsed,diskPUsage=getDiskUsage(TFILEDONEDIR)
0077
0078     except:
0079       doneSize=0
0080       diskSize,userAvailable,diskUsed,diskPUsage=getDiskUsage("/home")
0081
0082     diskPUsage*=100
0083     if diskPUsage < PRODUCER_DU_TOP:
0084       time.sleep(WAITTIME)
0085       continue
0086
0087     quota=int(diskSize*PRODUCER_DU_BOT/100)
0088     delQuota=diskUsed-quota
0089     if delQuota > doneSize:
0090       now = time.time()
0091       if now - EMAILINTERVAL > lastEmailSent:
0092         msg="ERROR: Something is filling up the disks, %s does not" \
0093           " have enough files to get to the Bottom Boundary of" \
0094           " %.2f%%" % (TFILEDONEDIR,PRODUCER_DU_BOT)
0095         sendmail(msg)
0096         lastEmailSent = now
0097
0098       logme("ERROR: Something is filling up the disks, %s does not" \
0099           " have enough files to get to the Bottom Boundary of" \
0100           " %.2f%%", TFILEDONEDIR, PRODUCER_DU_BOT)
0101
0102     aDelQuota=0
0103     FILE_LIST=[]
0104     for directory,subdirs,files in os.walk(TFILEDONEDIR):
0105       subdirs.sort()
0106       for f in sorted(files,key=lambda a: a[a.rfind("_R",1)+2:a.rfind("_R",1)+11]):
0107         fMatch=re.match(r"(DQM|Playback|Playback_full)_V[0-9]{4}_([0-9a-zA-Z]+)_R([0-9]{9})(_T[0-9]{8}|)\.root",f)
0108         if fMatch:
0109           subSystem=fMatch.group(2)
0110           run=fMatch.group(3)
0111           destDir="%s/%sxxxx/%sxx/DQM_V0001_%s_R%s.root" % (ORIGINALDONEDIR,run[0:5],run[0:7],subSystem,run)
0112           fullFName="%s/%s" % (directory,f)
0113           if os.stat(fullFName).st_size+aDelQuota > delQuota:
0114             break
0115
0116           FILE_LIST.append(fullFName)
0117           aDelQuota+=os.stat(fullFName).st_size
0118           if not os.path.exists(destDir):
0119             logme("WARNING: No subsystem file in repository %s for"
0120                   " file %s, deleting any way" %
0121                   (ORIGINALDONEDIR, fullFName))
0122
0123     if len(FILE_LIST):
0124       logme("INFO: Found %d files to be deleted", len(FILE_LIST))
0125
0126     #Cleanning ouput directory
0127     for directory,subdirs,files in os.walk(COLLECTDIR):
0128       #no subdiretories allowed in COLLECTDIR the  directory
0129       if subdirs:
0130         logme("ERROR: Output directory %s, must not contain"
0131               " subdirectories, cleanning", COLLECTDIR)
0132
0133       for sd in subdirs:
0134         fullSdName="%s/%s" % (directory,sd)
0135         for sdRoot,sdDirs,sdFiles in os.walk(fullSdName,topdown=False):
0136           for f in sdFiles:
0137             try:
0138               os.remove(f)
0139               logme("INFO: File %s has been removed", f)
0140             except Exception as e:
0141               logme("ERROR: Problem deleting file: [Errno %d] %s, '%s'",
0142                       e.errno, e.strerror, e.filename)
0143
0144           try:
0145             os.removedir(sdRoot)
0146             logme("INFO: File %s has been removed" , sdRoot)
0147           except Exception as e:
0148             logme("ERROR: Problem deleting directory: [Errno %d] %s, '%s'",
0149                       e.errno, e.strerror, e.filename)
0150
0151       for f in files:
0152         if re.match(r"(DQM|Playback|Playback_full)_V[0-9]{4}_([a-zA-Z]+)_R([0-9]{9})_T[0-9]{8}\.root", f):
0153           continue
0154
0155         if re.match(r".*\.tmp",f):
0156           continue
0157
0158         fullFName="%s/%s" % (directory, f)
0159         FILE_LIST.append(fullFName)
0160
0161       #cleaning tmp files:
0162       TMP_LIST=glob.glob("%s/*.tmp" % COLLECTDIR)
0163       TMP_LIST.sort(reverse=True,key=lambda x: os.stat(x).st_mtime)
0164       len(TMP_LIST) > 0 and TMP_LIST.pop(0)
0165       FILE_LIST.extend(TMP_LIST)
0166
0167     #remove files
0168     DIR_LIST=[]
0169     for f in FILE_LIST:
0170       try:
0171         os.remove(f)
0172         logme("INFO: File %s has been removed", f)
0173       except Exception as e:
0174         logme("ERROR: Problem deleting file: [Errno %d] %s, '%s'",
0175                 e.errno, e.strerror, e.filename)
0176       if os.path.dirname(f) not in DIR_LIST and COLLECTDIR not in os.path.dirname(f):
0177         DIR_LIST.append(os.path.dirname(f))
0178
0179     #remove emprty directories
0180     for d in DIR_LIST:
0181       try:
0182         os.removedirs(d)
0183         logme("INFO: Directory %s has been removed", d)
0184       except Exception as e:
0185         logme("ERROR: Directory delition failed: [Errno %d] %s, '%s'",
0186                 e.errno, e.strerror, e.filename)
0187
0188   except KeyboardInterrupt as e:
0189     sys.exit(0)
0190
0191   except Exception as e:
0192     logme('ERROR: %s', e)
0193     sendmail ('ERROR: %s\n%s' % (e, format_exc()))
0194     now = time.time()
0195     if now - EMAILINTERVAL > lastEmailSent:
0196       sendmail ('ERROR: %s\n%s' % (e, format_exc()))
0197       lastEmailSent = now
0198
0199     print_exc()
0200
0201   time.sleep(WAITTIME)
0202
0203
0204
0205