python/mpslib/Mpslibclass.py

0001 from __future__ import print_function
0002 # This Jobdatabas-Class interacts with the mps.db file.
0003 # It's member-variables are often called in the mps_... scripts.
0004 #
0005 # Meaning of the database variables: (still need to work on these)
0006 #
0007 # (1) Header
0008 #       header          - version information
0009 #       batchScript     - base script for serial job
0010 #       cfgTemplate     - template for cfg file
0011 #       infiList        - list of input files to be serialized
0012 #       classInf        - batch class information (might contain two ':'-separated)
0013 #       addFiles        - job name for submission
0014 #       driver          - specifies whether merge job is foreseen
0015 #       nJobs           - number of serial jobs (not including merge job)
0016 #       mergeScript     - base script for merge job
0017 #       mssDir          - directory for mass storage (e.g. Castor)
0018 #       updateTime      - time of last update (seconds since 1970)
0019 #       updateTimeHuman - time of last update (human readable)
0020 #       elapsedTime     - seconds since last update
0021 #       mssDirPool      - pool for $mssDir (e.g. cmscaf/cmscafuser)
0022 #       pedeMem         - Memory allocated for pede
0023 #       spare1
0024 #       spare2
0025 #       spare3
0026
0027 # (2) Job-level variables/lists
0028 #       JOBNUMBER   - ADDED, selfexplanatory
0029 #       JOBDIR      - name of job directory (not full path)
0030 #       JOBSTATUS   - status of job
0031 #       JOBRUNTIME  - present CPU time of job
0032 #       JOBNEVT     - number of events processed by job
0033 #       JOBHOST     - presently used to store remark
0034 #       JOBINCR     - CPU increment since last check
0035 #       JOBREMARK   - comment
0036 #       JOBSP1      - spare
0037 #       JOBSP2      - possible weight for pede
0038 #       JOBSP3      - possible name as given to mps_setup.pl -N <name> ...
0039 #       JOBID       - ID of the LSF/HTCondor job
0040
0041 from builtins import range
0042 import datetime
0043 import time
0044 import os
0045 import sys
0046
0047 #-------------------------------------------------------------------------------
0048 class jobdatabase:
0049
0050     JOBNUMBER, JOBDIR, JOBID, JOBSTATUS, JOBNTRY, JOBRUNTIME, JOBNEVT, JOBHOST, JOBINCR, \
0051     JOBREMARK, JOBSP1, JOBSP2, JOBSP3 = ([] for i in range(13))
0052
0053     header, batchScript, cfgTemplate, infiList, classInf, addFiles, driver, mergeScript, \
0054     mssDir, updateTimeHuman, mssDirPool, spare1, spare2, spare3 = ('' for i in range(14))
0055
0056     updateTime, elapsedTime, pedeMem , nJobs = -1, -1, -1, -1
0057
0058     #-------------------------------------------------------------------------------
0059     # parses the mps.db file into the member variables and arrays
0060     __default_db_file_name = "mps.db"
0061     def read_db(self, db_file_name = __default_db_file_name):
0062         try:
0063             DBFILE = open(db_file_name,'r')
0064         except IOError as e:
0065             if e.args != (2, 'No such file or directory'):
0066                 raise
0067             else:
0068                 if db_file_name == jobdatabase.__default_db_file_name:
0069                     msg = ("No 'mps.db' found. Make sure you are in a campaign "
0070                            "directory and that the campaign is set up.")
0071                 else:
0072                     msg = "Database file '"+db_file_name+"' not found. Exiting."
0073                 print(msg)
0074                 sys.exit(1)
0075
0076         #read infolines at the top, used rstrip to delete the '\n'
0077         self.header          = DBFILE.readline().strip()
0078         self.batchScript     = DBFILE.readline().rstrip('\n')
0079         self.cfgTemplate     = DBFILE.readline().rstrip('\n')
0080         self.infiList        = DBFILE.readline().rstrip('\n')
0081         self.classInf        = DBFILE.readline().rstrip('\n')   #formerly named 'class' ->conflict
0082         self.addFiles        = DBFILE.readline().rstrip('\n')
0083         self.driver          = DBFILE.readline().rstrip('\n')
0084         self.mergeScript     = DBFILE.readline().rstrip('\n')
0085         self.mssDir          = DBFILE.readline().rstrip('\n')
0086         self.updateTime      = int(DBFILE.readline())
0087         self.updateTimeHuman = DBFILE.readline().rstrip('\n')
0088         self.elapsedTime     = int(DBFILE.readline())
0089         self.mssDirPool      = DBFILE.readline().rstrip('\n')
0090         self.pedeMem         = int(DBFILE.readline())
0091         self.spare1          = DBFILE.readline().rstrip('\n')
0092         self.spare2          = DBFILE.readline().rstrip('\n')
0093         self.spare3          = DBFILE.readline().rstrip('\n')
0094
0095         #read actual jobinfo into arrays
0096         self.nJobs = 0
0097         milleJobs = 0
0098
0099
0100         for line in DBFILE:
0101             if line.strip() == "": continue # ignore empty lines
0102             line = line.rstrip('\n')        # removes the pesky \n from line
0103             parts = line.split(":")         # read each line and split into parts list
0104             self.JOBNUMBER.append(int(parts[0]))
0105             self.JOBDIR.append(parts[1].strip())
0106             self.JOBID.append(parts[2])
0107             self.JOBSTATUS.append(parts[3].strip())
0108             self.JOBNTRY.append(int(parts[4]))
0109             self.JOBRUNTIME.append(int(parts[5]))   #int float?
0110             self.JOBNEVT.append(int(parts[6]))
0111             self.JOBHOST.append(parts[7].strip())
0112             self.JOBINCR.append(int(parts[8]))
0113             self.JOBREMARK.append(parts[9].strip())
0114             self.JOBSP1.append(parts[10].strip())
0115             self.JOBSP2.append(parts[11].strip())
0116             self.JOBSP3.append(parts[12].strip())
0117
0118             #count number of jobs
0119             if not self.JOBDIR[self.nJobs].startswith("jobm"):
0120                 milleJobs += 1
0121             self.nJobs += 1
0122         self.nJobs = milleJobs
0123
0124         DBFILE.close()
0125
0126
0127
0128     #-------------------------------------------------------------------------------
0129     # prints the member varaiables and arrays to the terminal
0130     def print_memdb(self):
0131         #print metainfo
0132         print("\n=== mps database printout ===\n")
0133         print(self.header)
0134         print('Script:\t\t',    self.batchScript)
0135         print('cfg:\t\t',       self.cfgTemplate)
0136         print('files:\t\t',     self.infiList)
0137         print('class:\t\t',     self.classInf)
0138         print('name:\t\t',      self.addFiles)
0139         print('driver:\t\t',    self.driver)
0140         print('mergeScript:\t', self.mergeScript)
0141         print('mssDir:\t\t',    self.mssDir)
0142         print('updateTime:\t',  self.updateTimeHuman)
0143         print('elapsed:\t',     self.elapsedTime)
0144         print('mssDirPool:\t',  self.mssDirPool)
0145         print('pedeMem:\t',             self.pedeMem, '\n')
0146
0147         #print interesting Job-level lists ---- to add: t/evt, fix remarks
0148         print('###     dir      jobid    stat  try  rtime      nevt  remark   weight  name')
0149         print("------------------------------------------------------------------------------")
0150         for i in range(self.nJobs):
0151             print('%03d  %6s  %9s  %6s  %3d  %5d  %8d  %8s  %5s  %s' % (
0152                 self.JOBNUMBER[i],
0153                 self.JOBDIR[i],
0154                 self.JOBID[i],
0155                 self.JOBSTATUS[i],
0156                 self.JOBNTRY[i],
0157                 self.JOBRUNTIME[i],
0158                 self.JOBNEVT[i],
0159                 self.JOBHOST[i],
0160                 self.JOBSP2[i],
0161                 self.JOBSP3[i]))
0162
0163         #print merge Jobs if merge mode
0164         if self.driver == 'merge':
0165             for i in range(self.nJobs,len(self.JOBDIR)):
0166                 print('%s  %6s  %9s  %6s  %3d  %5d  %8d  %8s  %5s  %s' % (
0167                     'MMM',
0168                     self.JOBDIR[i],
0169                     self.JOBID[i],
0170                     self.JOBSTATUS[i],
0171                     self.JOBNTRY[i],
0172                     self.JOBRUNTIME[i],
0173                     self.JOBNEVT[i],
0174                     self.JOBHOST[i],
0175                     self.JOBSP2[i],
0176                     self.JOBSP3[i]))
0177
0178         #print summed info
0179         totalEvents = sum(self.JOBNEVT[:self.nJobs])
0180         totalCpu    = sum(self.JOBRUNTIME[:self.nJobs])
0181         meanCpuPerEvent = 0.
0182         if totalEvents > 0:
0183             meanCpuPerEvent = float(totalCpu)/totalEvents
0184         print("------------------------------------------------------------------------------")
0185         print("\t\t\t\t\tEvent total:\t",       totalEvents)
0186         print("\t\t\t\t\tCPU total:\t",         totalCpu,               's')
0187         print("\t\t\t\t\tMean CPU/event:\t",meanCpuPerEvent,'s')
0188
0189
0190
0191
0192
0193     #-------------------------------------------------------------------------------
0194     # writes a new mps.db file from the members. Replaces the old mps.db
0195     def write_db(self):
0196         self.header = "mps database schema 3.2"
0197         self.currentTime = int(time.time())
0198         self.elapsedTime = 0;
0199         if self.updateTime != 0:
0200             self.elapsedTime = self.currentTime - self.updateTime
0201         self.updateTime = self.currentTime
0202         self.updateTimeHuman = str(datetime.datetime.today())   #no timezone :(
0203         self.spare1 = "-- unused --"
0204         self.spare2 = "-- unused --"
0205         self.spare3 = "-- unused --"
0206
0207         #if mps.db already exists, backup as mps.db~ (in case of interupt during write)
0208         os.system('[[ -a mps.db ]] && cp -p mps.db mps.db~')
0209
0210         #write mps.db header
0211         DBFILE = open ("mps.db", "w")
0212         headData = [ self.header, self.batchScript, self.cfgTemplate, self.infiList,
0213                      self.classInf, self.addFiles, self.driver, self.mergeScript,
0214                      self.mssDir, self.updateTime, self.updateTimeHuman,
0215                      self.elapsedTime, self.mssDirPool, self.pedeMem,
0216                      self.spare1, self.spare2, self.spare3 ]
0217         for item in headData:
0218             DBFILE.write("%s\n" % item)
0219
0220         #write mps.db jobinfo
0221         for i in range(len(self.JOBID)):
0222             DBFILE.write('%03d:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s\n' %
0223                          (i+1,
0224                           self.JOBDIR[i],
0225                           self.JOBID[i],
0226                           self.JOBSTATUS[i],
0227                           self.JOBNTRY[i],
0228                           self.JOBRUNTIME[i],
0229                           self.JOBNEVT[i],
0230                           self.JOBHOST[i],
0231                           self.JOBINCR[i],
0232                           self.JOBREMARK[i],
0233                           self.JOBSP1[i],
0234                           self.JOBSP2[i],
0235                           self.JOBSP3[i]))
0236         DBFILE.close()
0237
0238     #-------------------------------------------------------------------------------
0239     # returns job class as stored in db
0240     # one and only argument may be "mille" or "pede" for mille or pede jobs
0241     def get_class(self, argument=''):
0242         CLASSES = self.classInf.split(':')
0243         if len(CLASSES)<1 or len(CLASSES)>2:
0244             print('\nget_class():\n  class must be of the form \'class\' or \'classMille:classPede\', but is \'%s\'!\n\n', classInf)
0245             sys.exit(1)
0246         elif argument == 'mille':
0247             return CLASSES[0]
0248         elif argument == 'pede':
0249             if len(CLASSES) == 1:
0250                 return CLASSES[0]
0251             elif len(CLASSES) == 2:
0252                 return CLASSES[1]
0253         else:
0254             print('\nget_class():\n  Know class only for \'mille\' or \'pede\', not %s!\n\n' %argument)
0255             sys.exit(1)