scripts/mpslib/Mpslib.pm

0001 package Mpslib;  # assumes Some/Module.pm
0002
0003 # $Revision: 1.9 $ by $Author: jbehr $
0004 #
0005 # Meaning of the database variables:
0006 #
0007 # (1) Header
0008 #
0009 #  $header  - version information
0010 #  $batchScript - base script for serial job
0011 #  $cfgTemplate - template for cfg file
0012 #  $infiList  -  list of input files to be serialized
0013 #  $class - batch class information (might contain two ':'-separated)
0014 #  $addFiles - job name for submission
0015 #  $driver - specifies whether merge job is foreseen
0016 #  $nJobs - number of serial jobs (not including merge job)
0017 #  $mergeScript - base script for merge job
0018 #  $mssDir - directory for mass storage (e.g. Castor)
0019 #  $updateTime - time of last update (seconds since 1970)
0020 #  $updateTimeHuman - time of last update (human readable)
0021 #  $elapsedTime - seconds since last update
0022 #  $mssDirPool - pool for $mssDir (e.g. cmscaf/cmscafuser)
0023 #  $pedeMem - Memory allocated for pede
0024 #  $spare1
0025 #  $spare2
0026 #  $spare3
0027 #
0028 # (2) Job-level variables
0029 #
0030 #  @JOBDIR  - name of job directory (not full path)
0031 #  @JOBSTATUS - status of job
0032 #  @JOBRUNTIME -  present CPU time of job
0033 #  @JOBNEVT - number of events processed by job
0034 #  @JOBHOST - presently used to store remark
0035 #  @JOBINCR - CPU increment since last check
0036 #  @JOBREMARK - comment
0037 #  @JOBSP1 - spare
0038 #  @JOBSP2 - possible weight for pede
0039 #  @JOBSP3 - possible name as given to mps_setup.pl -N <name> ...
0040
0041
0042   use Exporter   ();
0043   use vars       qw( $VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
0044 #  use DBI;
0045   # set the version for version checking
0046   $VERSION     = 1.01;
0047
0048   @ISA         = qw(Exporter);
0049   @EXPORT      = qw(
0050                     write_db
0051                     read_db
0052                     print_memdb
0053             get_class
0054             @JOBID
0055             $header
0056             $batchScript $cfgTemplate $infiList $class $addFiles $driver $nJobs
0057                     $mergeScript $mssDir $updateTime $updateTimeHuman $elapsedTime $mssDirPool $pedeMem $spare1 $spare2 $spare3
0058             @JOBDIR
0059             @JOBSTATUS @JOBNTRY @JOBRUNTIME @JOBNEVT @JOBHOST @JOBINCR @JOBREMARK @JOBSP1 @JOBSP2 @JOBSP3
0060                    );
0061
0062 our $eos = "/afs/cern.ch/project/eos/installation/cms/bin/eos.select";
0063
0064 sub write_db() {
0065   $header = "mps database schema 3.2" ;
0066   $currentTime = `date +%s`;
0067   chomp $currentTime;
0068   $elapsedTime = 0;
0069   if ($updateTime != 0) { $elapsedTime = $currentTime - $updateTime; }
0070   $updateTime = $currentTime;
0071   $updateTimeHuman = `date`;
0072   chomp $updateTimeHuman;
0073   $spare1 = "-- unused --";
0074   $spare2 = "-- unused --";
0075   $spare3 = "-- unused --";
0076
0077   system "[[ -a mps.db ]] && cp -p mps.db mps.db~"; # GF: backup if exists (in case of interupt during write)
0078   open DBFILE,">mps.db";
0079   printf DBFILE "%s\n",$header;
0080   printf DBFILE "%s\n",$batchScript;
0081   printf DBFILE "%s\n",$cfgTemplate;
0082   printf DBFILE "%s\n",$infiList;
0083   printf DBFILE "%s\n",$class;
0084   printf DBFILE "%s\n",$addFiles;
0085   printf DBFILE "%s\n",$driver;
0086   printf DBFILE "%s\n",$mergeScript;
0087   printf DBFILE "%s\n",$mssDir;
0088   printf DBFILE "%d\n",$updateTime;
0089   printf DBFILE "%s\n",$updateTimeHuman;
0090   printf DBFILE "%d\n",$elapsedTime;
0091   printf DBFILE "%s\n",$mssDirPool;
0092   printf DBFILE "%d\n",$pedeMem;
0093   printf DBFILE "%s\n",$spare1;
0094   printf DBFILE "%s\n",$spare2;
0095   printf DBFILE "%s\n",$spare3;
0096   my $i;
0097   for ($i = 0; $i < @JOBID; ++$i) {
0098     printf DBFILE "%03d:%s:%s:%s:%d:%d:%d:%s:%d:%s:%s:%s:%s\n",
0099     $i+1,@JOBDIR[$i],@JOBID[$i],@JOBSTATUS[$i],@JOBNTRY[$i],@JOBRUNTIME[$i],@JOBNEVT[$i],@JOBHOST[$i],
0100     @JOBINCR[$i],@JOBREMARK[$i],@JOBSP1[$i],@JOBSP2[$i],@JOBSP3[$i];
0101   }
0102   close DBFILE;
0103 }
0104
0105 sub read_db() {
0106   open DBFILE,"mps.db";
0107   @JOBDIR = ();
0108   @JOBID= ();
0109   @JOBSTATUS = ();
0110   @JOBNTRY = ();
0111   @JOBRUNTIME = ();
0112   @JOBNEVT = ();
0113   @JOBHOST = ();
0114   @JOBINCR = ();
0115   @JOBREMARK = ();
0116   @JOBSP1 = ();
0117   @JOBSP2 = ();
0118   @JOBSP3 = ();
0119
0120   $header = <DBFILE>;
0121   $batchScript = <DBFILE>;
0122   $cfgTemplate = <DBFILE>;
0123   $infiList = <DBFILE>;
0124   $class = <DBFILE>;
0125   $addFiles = <DBFILE>;
0126   $driver = <DBFILE>;
0127   $mergeScript = <DBFILE>;
0128   $mssDir = <DBFILE>;
0129   $updateTime = <DBFILE>;
0130   $updateTimeHuman = <DBFILE>;
0131   $elapsedTime = <DBFILE>;
0132   $mssDirPool = <DBFILE>;
0133   $pedeMem = <DBFILE>;
0134   $spare1 = <DBFILE>;
0135   $spare2 = <DBFILE>;
0136   $spare3 = <DBFILE>;
0137   chomp $header;
0138   chomp $batchScript;
0139   chomp $cfgTemplate;
0140   chomp $infiList;
0141   chomp $class;
0142   chomp $addFiles;
0143   chomp $driver;
0144   chomp $mergeScript;
0145   chomp $mssDir;
0146   chomp $updateTime;
0147   chomp $updateTimeHuman;
0148   chomp $elapsedTime;
0149   chomp $mssDirPool;
0150   chomp $pedeMem;
0151   chomp $spare1;
0152   chomp $spare2;
0153   chomp $spare3;
0154
0155   my $nMilleJobs = 0;
0156   $nJobs = 0;
0157   while (<DBFILE>) { # loop through all jobs to read
0158     #chomp $_;
0159     my $line;
0160     my $nsplit = ($line,@JOBDIR[$nJobs],@JOBID[$nJobs],@JOBSTATUS[$nJobs],
0161     @JOBNTRY[$nJobs],@JOBRUNTIME[$nJobs],@JOBNEVT[$nJobs],@JOBHOST[$nJobs],@JOBINCR[$nJobs],
0162     @JOBREMARK[$nJobs],@JOBSP1[$nJobs],@JOBSP2[$nJobs],@JOBSP3[$nJobs])
0163       = split(":",$_);
0164     chomp $JOBSP3[$nJobs];
0165     unless (@JOBDIR[$nJobs] =~ m/jobm/) { # count mille jobs
0166     ++$nMilleJobs;
0167     }
0168     ++$nJobs;
0169   }
0170   $nJobs=$nMilleJobs;
0171   close DBFILE;
0172 }
0173
0174 sub print_memdb() {
0175   print "=== mps database printout ===\n";
0176   print "$header\n";
0177   printf "Script: %s\ncfg: %s\nfiles: %s\nclass: %s\nname: %s\ndriver: %s\nmergeScript: %s\nmssDir: %s\nupdateTime: %s\nelapsed: %d\nmssDirPool: %s\npedeMem: %d\n",$batchScript,
0178   $cfgTemplate,$infiList,$class,$addFiles,$driver,$mergeScript,$mssDir,$updateTimeHuman,$elapsedTime,$mssDirPool,$pedeMem;
0179   printf "%3s %6s %9s %6s %3s %5s %8s %6s %8s %s\n",
0180   '###',"dir","jobid","stat","try","rtime","nevt","t/evt","remark","name";
0181   my $i;
0182   my $totEvt = 0;
0183   my $totCpu = 0;
0184   for ($i=0; $i<$nJobs; ++$i) {
0185     $cpuFactor = get_cpufactor(@JOBHOST[$i]);
0186     $thisCpu = @JOBRUNTIME[$i] * $cpuFactor;
0187     my $cpuPerEvt = 0;
0188     if (@JOBRUNTIME[$i]>0 and @JOBNEVT[$i]>0) {
0189       $cpuPerEvt = $thisCpu / @JOBNEVT[$i];
0190     }
0191     printf "%03d %6s %9s %6s %3d %5d %8d %6.3f %8s %s\n",
0192     $i+1,@JOBDIR[$i],@JOBID[$i],@JOBSTATUS[$i],@JOBNTRY[$i],
0193       $thisCpu,@JOBNEVT[$i],$cpuPerEvt,@JOBHOST[$i],$JOBSP3[$i];
0194     if (@JOBNEVT[$i] > 0) {
0195       $totEvt = $totEvt + @JOBNEVT[$i];
0196     }
0197     $totCpu = $totCpu + $thisCpu;
0198   }
0199
0200   # if merge mode, print merge job(s) as well
0201   if ($driver eq "merge") {
0202       $i = $nJobs;
0203       while ($i < @JOBID) {
0204         $cpuFactor = get_cpufactor(@JOBHOST[$i]);
0205         $thisCpu = @JOBRUNTIME[$i] * $cpuFactor;
0206         printf "%3s %6s %9s %6s %3d %5d %8d %6.3f %8s\n",
0207         "MMM",@JOBDIR[$i],@JOBID[$i],@JOBSTATUS[$i],@JOBNTRY[$i],
0208         $thisCpu,@JOBNEVT[$i],0,@JOBHOST[$i];
0209     ++$i;
0210       }
0211   }
0212
0213   my $meanCpuPerEvt = 0;
0214   if ($totEvt>0) {
0215     $meanCpuPerEvt = $totCpu / $totEvt;
0216   }
0217   print "----------------------------------------------\n";
0218   printf "                       Event total: %10d\n",$totEvt;
0219   printf "                         CPU total: %10.1f s\n",$totCpu;
0220   printf "                    Mean CPU/event: %10.3f s\n",$meanCpuPerEvt;
0221 }
0222
0223 sub get_class {
0224     # returns job class as stored in db
0225     # one and only argument may be "mille" or "pede" for mille or pede jobs
0226     my @CLASSES = split ":",$class;
0227     if (@CLASSES < 1 || @CLASSES > 2) {
0228     print "\nget_class():\n  class must be of the form 'class' or 'classMille:classPede', but is '$class'!\n\n";
0229     exit 1;
0230     return "";
0231     } elsif ($_[0] eq "mille") {
0232     return $CLASSES[0];
0233     } elsif ($_[0] eq "pede") {
0234     if (@CLASSES == 1) {
0235         return $CLASSES[0];
0236     } elsif (@CLASSES == 2) {
0237         return $CLASSES[1];
0238     }
0239     } else {
0240     print "\nget_class():\n  Know class only for 'mille' or 'pede', not $_[0]!\n\n";
0241     exit 1;
0242     return "";
0243     }
0244 }
0245
0246 sub get_cpufactor() {
0247   # $hostName = $_[0];
0248   # $znumber = 0;
0249   # my $cf;
0250   # if ( ($hostName =~ m/zenith(\d+)/) == 1) {
0251   #   $znumber = $1;
0252   # }
0253   # if ($znumber >=40 && $znumber < 100) {
0254   #   $cf = 1.;
0255   # }
0256   # elsif ($znumber >= 100 && $znumber < 155) {
0257   #   $cf = 3.06/2.2;
0258   # }
0259   # elsif ($znumber >= 155) {
0260   #   $cf = 1.25*3.06/2.2;
0261   # }
0262   # else {
0263   #   $cf = 1.;
0264   # }
0265   # $theFactor = $cf;
0266   $theFactor = 1;
0267 }