File indexing completed on 2024-11-26 02:34:08
0001
0002 import os, time, sys, glob, re, smtplib, socket
0003 from email.MIMEText import MIMEText
0004 from traceback import print_exc, format_exc
0005 from datetime import datetime
0006 from subprocess import Popen,PIPE
0007 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
0008
0009 EMAIL = sys.argv[1]
0010 TFILEDONEDIR = sys.argv[2]
0011 COLLECTDIR = sys.argv[3]
0012 ORIGINALDONEDIR =sys.argv[4]
0013
0014
0015 PRODUCER_DU_TOP= 90.0
0016 PRODUCER_DU_BOT= 50.0
0017 WAITTIME = 3600 * 4
0018 EMAILINTERVAL = 15 * 60
0019 SENDMAIL = "/usr/sbin/sendmail"
0020 HOSTNAME = socket.gethostname().lower()
0021 EXEDIR = os.path.dirname(__file__)
0022 STOP_FILE = "%s/.stop" % EXEDIR
0023
0024
0025 lastEmailSent = 0
0026
0027
0028 def logme(msg, *args):
0029 procid = "[%s/%d]" % (__file__.rsplit("/", 1)[-1], os.getpid())
0030 print(datetime.now(), procid, msg % args)
0031
0032 def getDiskUsage(path):
0033 fsStats=os.statvfs(path)
0034 size=fsStats.f_bsize*fsStats.f_blocks
0035 available=fsStats.f_bavail*fsStats.f_bsize
0036 used=size-available
0037 usedPer=float(used)/size
0038 return (size,available,used,usedPer)
0039
0040 def getDirSize(path):
0041 import stat
0042 size=os.stat(path).st_blksize
0043 for directory,subdirs,files in os.walk(path):
0044 dStats=os.lstat(directory)
0045 size+=(dStats[stat.ST_NLINK]-1)*dStats[stat.ST_SIZE]
0046 for f in files:
0047 fStats=os.lstat("%s/%s" % (directory,f))
0048 fSize=fStats[stat.ST_SIZE]
0049 size+=fSize
0050
0051 return size
0052
0053 def sendmail(body="Hello from producerFileCleanner",subject= "Hello!"):
0054 scall = Popen("%s -t" % SENDMAIL, shell=True, stdin=PIPE)
0055 scall.stdin.write("To: %s\n" % EMAIL)
0056 scall.stdin.write("Subject: producerFileCleaner problem on server %s\n" %
0057 HOSTNAME)
0058 scall.stdin.write("\n")
0059 scall.stdin.write("%s\n" % body)
0060 scall.stdin.close()
0061 rc = scall.wait()
0062 if rc != 0:
0063 logme("ERROR: Sendmail exit with status %s", rc)
0064
0065
0066 while True:
0067
0068 if os.path.exists(STOP_FILE):
0069 logme("INFO: Stop file found, quitting")
0070 sys.exit(0)
0071
0072 try:
0073 try:
0074 doneSize=getDirSize(TFILEDONEDIR)
0075 diskSize,userAvailable,diskUsed,diskPUsage=getDiskUsage(TFILEDONEDIR)
0076
0077 except:
0078 doneSize=0
0079 diskSize,userAvailable,diskUsed,diskPUsage=getDiskUsage("/home")
0080
0081 diskPUsage*=100
0082 if diskPUsage < PRODUCER_DU_TOP:
0083 time.sleep(WAITTIME)
0084 continue
0085
0086 quota=int(diskSize*PRODUCER_DU_BOT/100)
0087 delQuota=diskUsed-quota
0088 if delQuota > doneSize:
0089 now = time.time()
0090 if now - EMAILINTERVAL > lastEmailSent:
0091 msg="ERROR: Something is filling up the disks, %s does not" \
0092 " have enough files to get to the Bottom Boundary of" \
0093 " %.2f%%" % (TFILEDONEDIR,PRODUCER_DU_BOT)
0094 sendmail(msg)
0095 lastEmailSent = now
0096
0097 logme("ERROR: Something is filling up the disks, %s does not" \
0098 " have enough files to get to the Bottom Boundary of" \
0099 " %.2f%%", TFILEDONEDIR, PRODUCER_DU_BOT)
0100
0101 aDelQuota=0
0102 FILE_LIST=[]
0103 for directory,subdirs,files in os.walk(TFILEDONEDIR):
0104 subdirs.sort()
0105 for f in sorted(files,key=lambda a: a[a.rfind("_R",1)+2:a.rfind("_R",1)+11]):
0106 fMatch=re.match(r"(DQM|Playback|Playback_full)_V[0-9]{4}_([0-9a-zA-Z]+)_R([0-9]{9})(_T[0-9]{8}|)\.root",f)
0107 if fMatch:
0108 subSystem=fMatch.group(2)
0109 run=fMatch.group(3)
0110 destDir="%s/%sxxxx/%sxx/DQM_V0001_%s_R%s.root" % (ORIGINALDONEDIR,run[0:5],run[0:7],subSystem,run)
0111 fullFName="%s/%s" % (directory,f)
0112 if os.stat(fullFName).st_size+aDelQuota > delQuota:
0113 break
0114
0115 FILE_LIST.append(fullFName)
0116 aDelQuota+=os.stat(fullFName).st_size
0117 if not os.path.exists(destDir):
0118 logme("WARNING: No subsystem file in repository %s for"
0119 " file %s, deleting any way" %
0120 (ORIGINALDONEDIR, fullFName))
0121
0122 if len(FILE_LIST):
0123 logme("INFO: Found %d files to be deleted", len(FILE_LIST))
0124
0125
0126 for directory,subdirs,files in os.walk(COLLECTDIR):
0127
0128 if subdirs:
0129 logme("ERROR: Output directory %s, must not contain"
0130 " subdirectories, cleanning", COLLECTDIR)
0131
0132 for sd in subdirs:
0133 fullSdName="%s/%s" % (directory,sd)
0134 for sdRoot,sdDirs,sdFiles in os.walk(fullSdName,topdown=False):
0135 for f in sdFiles:
0136 try:
0137 os.remove(f)
0138 logme("INFO: File %s has been removed", f)
0139 except Exception as e:
0140 logme("ERROR: Problem deleting file: [Errno %d] %s, '%s'",
0141 e.errno, e.strerror, e.filename)
0142
0143 try:
0144 os.removedir(sdRoot)
0145 logme("INFO: File %s has been removed" , sdRoot)
0146 except Exception as e:
0147 logme("ERROR: Problem deleting directory: [Errno %d] %s, '%s'",
0148 e.errno, e.strerror, e.filename)
0149
0150 for f in files:
0151 if re.match(r"(DQM|Playback|Playback_full)_V[0-9]{4}_([a-zA-Z]+)_R([0-9]{9})_T[0-9]{8}\.root", f):
0152 continue
0153
0154 if re.match(r".*\.tmp",f):
0155 continue
0156
0157 fullFName="%s/%s" % (directory, f)
0158 FILE_LIST.append(fullFName)
0159
0160
0161 TMP_LIST=glob.glob("%s/*.tmp" % COLLECTDIR)
0162 TMP_LIST.sort(reverse=True,key=lambda x: os.stat(x).st_mtime)
0163 len(TMP_LIST) > 0 and TMP_LIST.pop(0)
0164 FILE_LIST.extend(TMP_LIST)
0165
0166
0167 DIR_LIST=[]
0168 for f in FILE_LIST:
0169 try:
0170 os.remove(f)
0171 logme("INFO: File %s has been removed", f)
0172 except Exception as e:
0173 logme("ERROR: Problem deleting file: [Errno %d] %s, '%s'",
0174 e.errno, e.strerror, e.filename)
0175 if os.path.dirname(f) not in DIR_LIST and COLLECTDIR not in os.path.dirname(f):
0176 DIR_LIST.append(os.path.dirname(f))
0177
0178
0179 for d in DIR_LIST:
0180 try:
0181 os.removedirs(d)
0182 logme("INFO: Directory %s has been removed", d)
0183 except Exception as e:
0184 logme("ERROR: Directory delition failed: [Errno %d] %s, '%s'",
0185 e.errno, e.strerror, e.filename)
0186
0187 except KeyboardInterrupt as e:
0188 sys.exit(0)
0189
0190 except Exception as e:
0191 logme('ERROR: %s', e)
0192 sendmail ('ERROR: %s\n%s' % (e, format_exc()))
0193 now = time.time()
0194 if now - EMAILINTERVAL > lastEmailSent:
0195 sendmail ('ERROR: %s\n%s' % (e, format_exc()))
0196 lastEmailSent = now
0197
0198 print_exc()
0199
0200 time.sleep(WAITTIME)
0201
0202
0203
0204