CommonAlignmentProducer/test/alignment.pl

0001 #!/usr/bin/perl
0002 ###############################################################################
0003 # Parallel processing for alignment
0004 ###############################################################################
0005
0006 # determine location
0007 $host=`echo \$HOST`;
0008 if    ( $host =~ /fpslife/ ) { $location="fpslife"; }
0009 elsif ( $host =~ /lxplus/ )  { $location="lxplus"; }
0010 elsif ( $host =~ /cnaf/ )    { $location="cnaf"; }
0011 elsif ( $host =~ /lxcms/ )  { $location="lxplus"; }
0012
0013 # job steering ----------------------------------------------------------------
0014
0015 # name of job
0016 $jobname="test";
0017
0018 # cfg file
0019 $steering="alignment.cfg";
0020
0021 # db output files to be deleted except for last iteration
0022 $dbfiles="Alignments.db condbcatalog.xml";
0023
0024 # db authentication file to be copied to exec dir
0025 $authfile="authentication.xml";
0026
0027 # db input file
0028 # $sqlitefile="CSA06Scenario.db"; $condbcatalogfile="condbcatalog.xml";
0029 $sqlitefile=""; $condbcatalogfile="";
0030
0031 # number of events per job
0032 $nevent=10000;
0033
0034 # first event
0035 $firstev=0;
0036
0037 # number of jobs
0038 $njobs=1;
0039
0040 # number of iterations (excluding initial step)
0041 $iterations=3;
0042
0043 # interactive or lxbatch queue
0044 $farm="I";
0045 # $farm="8nm"; $resource="";
0046 # $farm="dedicated -R cmsalca";
0047
0048 $cmsswvers="CMSSW_1_2_0_pre3";
0049 $scramarch="slc3_ia32_gcc323";
0050 $scram=scramv1;
0051
0052 # sleep time in seconds between two check cycles
0053 $sleeptime=30;
0054
0055 # site-specific paths etc
0056
0057 if ( $location eq "lxplus" ) {
0058   $homedir="/afs/cern.ch/user/f/fpschill";
0059   $basedir="${homedir}/w1/cmssw/${cmsswvers}";
0060   $outdir="${homedir}/scratch0/joboutput";
0061 }
0062 elsif ( $location eq "fpslife" ) {
0063    die "ERROR: location: $location unsupported!\n";
0064 #  $homedir="/afs/cern.ch/user/f/fpschill";
0065 #  $basedir="${homedir}/cms/${cmsswvers}";
0066 #  $outdir="/x01/usr/fpschill/cms/joboutput";
0067 }
0068 elsif ( $location eq "cnaf" ) {
0069    die "ERROR: location: $location unsupported!\n";
0070 #  $homedir="/home/CMS/fpschill";
0071 #  $basedir="${homedir}/${cmsswvers}";
0072 #  $outdir="/afs/infn.it/cnaf/user/fpschill/joboutput";
0073 }
0074 else {
0075   die "ERROR: location: $location unknown!\n";
0076 }
0077
0078 # -----------------------------------------------------------------------------
0079
0080 $workdir="${basedir}/src/Alignment/CommonAlignmentProducer/test";
0081
0082 print "\n";
0083 print "-------------------------------------------------------------------------------\n";
0084 print "A l i g n m e n t \n";
0085 print "-------------------------------------------------------------------------------\n";
0086 print "Location: $location\n";
0087 print "Workdir: ${workdir}\n";
0088 print "Outdir: ${outdir}\n";
0089 if ($njobs>1) {
0090   print "Parallel Jobs: ${njobs} with $nevent events per job\n";
0091 }
0092 else {
0093   print "Events: $nevent \n";
0094 }
0095 print "Iterations: ${iterations}\n";
0096
0097 $odir="$outdir/$jobname";
0098
0099 # -----------------------------------------------------------------------------
0100 # create sandbox and set up local environment
0101
0102 system("
0103 rm -rf $outdir/$jobname;
0104 mkdir $outdir/$jobname;
0105 cd $outdir/$jobname;
0106 $scram project CMSSW $cmsswvers > /dev/null;
0107 cd $cmsswvers;
0108 mkdir -p lib/${scramarch};
0109 mkdir -p bin/${scramarch};
0110 mkdir -p config/${scramarch};
0111 cp $basedir/lib/${scramarch}/* lib/${scramarch} > /dev/null;
0112 cp $basedir/module/${scramarch}/* module/${scramarch} > /dev/null;
0113 ");
0114
0115 # deal with modified cf? files
0116 chdir "$basedir/src";
0117 @cfgfilelist = `ls */*/*/*.cf?`;
0118 foreach $file ( @cfgfilelist ) {
0119   system("tar rf cfg.tar $file");
0120 }
0121 system("cp cfg.tar $odir/$cmsswvers/src; rm cfg.tar; cd $odir/$cmsswvers/src; tar xf cfg.tar");
0122
0123
0124 # if one cpu create subjob and submit #########################################
0125
0126 if ($njobs == 1) {
0127
0128   # make subjob
0129   $dir="$outdir/$jobname/$cmsswvers/main";
0130   make_subjob($dir,1);
0131
0132   # create cfg file
0133   system("cp ${workdir}/$steering $dir/cfgfile");
0134   $repl="replace maxEvents  = { untracked int32 input = $nevent }";
0135   replace("$dir/cfgfile",$repl);
0136
0137   if ($farm eq "I") {
0138     print "Run interactively ...\n";
0139     system("cd $dir; ./subjob");
0140   }
0141   else {
0142     print "Submit to farm ...\n";
0143     system("bsub -i /tmp/junk -q $farm < $dir/subjob");
0144   }
0145
0146 }
0147
0148 # PARALLEL ####################################################################
0149
0150 else {
0151
0152 # -----------------------------------------------------------------------------
0153 # create N job directories and set up steering
0154
0155 $ijob=1;
0156 $ifirst=$firstev;
0157
0158 while ( ${ijob} <= ${njobs} ) {
0159   print "Job ${ijob}/${njobs} starting at event $ifirst \n";
0160
0161   # make job dir and copy aux files
0162   $dir="$outdir/$jobname/$cmsswvers/job$ijob";
0163   make_subjob($dir,0);
0164
0165   # create cfg file
0166   system("cp ${workdir}/$steering $dir/cfgfile");
0167   $repl="
0168     replace PoolSource.maxEvents  = $nevent
0169     replace PoolSource.skipEvents = $ifirst
0170     replace AlignmentProducer.saveToDB = false
0171   ";
0172   replace("$dir/cfgfile",$repl);
0173
0174   ${ijob}++;
0175   ${ifirst}=${ifirst}+${nevent};
0176 };
0177
0178 # -----------------------------------------------------------------------------
0179 # create collector job and steering
0180
0181 $dir="$outdir/$jobname/$cmsswvers/main";
0182 make_subjob($dir,0);
0183
0184 # create cfg file
0185 system("cp ${workdir}/$steering $dir/cfgfile");
0186 $repl="
0187   replace PoolSource.maxEvents  = 1
0188   replace PoolSource.skipEvents = 0
0189   replace HIPAlignmentAlgorithm.collectorActive = true
0190   replace HIPAlignmentAlgorithm.collectorNJobs = $njobs
0191   replace HIPAlignmentAlgorithm.collectorPath = \"../\"
0192 ";
0193 replace("$dir/cfgfile",$repl);
0194
0195
0196 # -----------------------------------------------------------------------------
0197
0198 $dir="$outdir/$jobname/$cmsswvers";
0199 $iteration=0;
0200
0201 system("rm -f $dir/iteration.txt");
0202 open(FILE,">$dir/iteration.txt");
0203 print FILE "$iteration";
0204 close(FILE);
0205
0206 # enter loop ... ==============================================================
0207
0208 LOOP: {
0209
0210   # check if all are finished
0211   if ($iteration gt 0 ) { $alldone=&check_finished(); }
0212
0213   # need to run collector / resubmit for next iteration
0214   if ( $alldone eq 1 or $iteration eq 0) {
0215
0216     # run collector
0217     print "Run collector for iteration $iteration ...\n";
0218     $iret=&run_collector();
0219     if ($iret ne 0) { die "ERROR in collector!\n";}
0220
0221     # submit jobs for next iteration
0222     if ($iteration < $iterations) {
0223       $iteration++;
0224
0225       system("rm -f $dir/iteration.txt");
0226       open(FILE,">$dir/iteration.txt");
0227       print FILE "$iteration";
0228       close(FILE);
0229
0230       print "New iteration $iteration --------------------------------------------------------------\n";
0231       print "Submit all jobs for iteration $iteration ...\n";
0232       $iret=&submit_jobs;
0233       if ($iret ne 0) { die "ERROR in submit jobs!\n";}
0234
0235       redo LOOP;
0236     }
0237     # we are done
0238     else {
0239       system("
0240         cd $dir
0241         rm -rf tmp src logs lib config bin
0242      ");
0243       print "Finished!\n";
0244     }
0245
0246   }
0247
0248   # else, just wait ...
0249   else {
0250     # print "sleeping for $sleeptime seconds ...\n";
0251     sleep($sleeptime);
0252     redo LOOP;
0253   }
0254
0255 }
0256
0257 }
0258
0259 exit 0;
0260
0261 ###############################################################################
0262
0263 sub check_finished
0264 {
0265
0266 $alldone=1;
0267 $ijob=1;
0268
0269 $timestamp=`date +%H:%M:%S_%d/%m/%y`;
0270
0271 print "Job status: ";
0272
0273 while ( $ijob <= $njobs ) {
0274   if ( -e "$dir/job$ijob/DONE" ) { print "1 "; }
0275   else { print "0 "; $alldone=0; }
0276   $ijob++;
0277 }
0278
0279 print "at $timestamp";
0280 if ($alldone eq 1) { return 1;}
0281 else { return 0; }
0282
0283
0284 }
0285
0286 ###############################################################################
0287
0288 sub run_collector
0289 {
0290   system("
0291     cd $dir/main
0292     ./subjob
0293   ");
0294   if ($iteration gt 0) {
0295     system("cp $dir/job1/HIPAlignmentEvents.root $dir/main");
0296   }
0297
0298   return 0;
0299 }
0300
0301 ###############################################################################
0302
0303 sub submit_jobs
0304 {
0305
0306 $ijob=1;
0307 while ( $ijob <= $njobs ) {
0308
0309   system("
0310     rm -f $dir/job$ijob/DONE
0311
0312     cp $dir/main/IOTruePositions.root       $dir/job$ijob
0313     cp $dir/main/IOMisalignedPositions.root $dir/job$ijob
0314     cp $dir/main/IOAlignedPositions.root    $dir/job$ijob
0315     cp $dir/main/IOIteration.root           $dir/job$ijob
0316
0317   ");
0318
0319   if ( $farm eq "I" ) {
0320     print "Run job $ijob interactively ...\n";
0321     system("$dir/job$ijob/subjob &");
0322   }
0323   else {
0324     print "Submit job $ijob ... ";
0325     $test=0;
0326     while($test == 0) {
0327       $rc=system("cd $dir ; bsub -o /tmp/junk -q $farm < job$ijob/subjob");
0328       if ($rc == 0) { $test=1; }
0329       else {
0330     print "ERROR in submitting job: $rc  .. retrying in 10s ...\n";
0331     sleep(5);
0332       }
0333     }
0334   }
0335   sleep(1);
0336   $ijob++;
0337 }
0338
0339   return 0;
0340 }
0341
0342 ###############################################################################
0343
0344 sub make_subjob {
0345
0346 $dir=@_[0];
0347 $single=@_[1];
0348
0349 system("mkdir $dir");
0350 system("cp ${workdir}/$authfile $dir");
0351 if ($sqlitefile !="") { system("cp ${workdir}/$sqlitefile $dir"); }
0352 if ($condbcatalogfile !="") { system("cp ${workdir}/$condbcatalogfile $dir"); }
0353
0354 if ($single == 1) {
0355
0356 # for single job
0357 system("touch $dir/timing.log");
0358 $subjob="#!/bin/zsh -f
0359 #BSUB -J \"ALIGN\"
0360 #BSUB -C 0
0361 cd $dir
0362 eval \`$scram runtime -sh\`
0363 rehash
0364 thisiter=1
0365 until ((thisiter > $iterations )); do
0366 rm -f ${dbfiles}
0367 echo \"Iteration \${thisiter} ...\"
0368 date | read starttime
0369 cmsRun cfgfile >& log.\${thisiter}.log
0370 date | read endtime
0371 gzip log.\${thisiter}.log
0372 mv alignment.log alignment.\${thisiter}.log
0373 gzip alignment.\${thisiter}.log
0374 echo \"Iteration \${thisiter}: \${starttime} ... \${endtime}\" >> timing.log
0375 let thisiter++
0376 done
0377 ";
0378
0379 }
0380
0381 else {
0382
0383 # for parallel
0384 $subjob="#!/bin/zsh
0385 #BSUB -J \"ALIGN\"
0386 #BSUB -C 0
0387 cd $dir
0388 eval \`$scram runtime -sh\`
0389 rehash
0390 cat ../iteration.txt | read iter
0391 rm -f $dbfiles
0392 cmsRun cfgfile >& log.\$iter.log
0393 gzip log.\$iter.log
0394 mv alignment.log alignment.\$iter.log
0395 gzip alignment.\$iter.log
0396 touch DONE
0397 ";
0398
0399 }
0400
0401 open(FILE,">$dir/subjob");
0402 print FILE "$subjob";
0403 close(FILE);
0404 system("chmod u+x $dir/subjob");
0405
0406 }
0407
0408 ###############################################################################
0409
0410 sub replace {
0411
0412 $infile = @_[0];
0413 $repl = @_[1];
0414
0415 open(INFILE,"$infile") or die "cannot open $infile";;
0416 @log=<INFILE>;
0417 close(INFILE);
0418
0419 system("rm -f tmp");
0420 open(OUTFILE,">tmp");
0421
0422 foreach $line (@log) {
0423   if ($line =~ /REPLACEME/) { print OUTFILE $repl; }
0424   else { print OUTFILE $line; }
0425 }
0426
0427 close(OUTFILE);
0428 system("mv tmp $infile");
0429
0430 }