L1TCommon/scripts/testL1T.pl

0001 #! /bin/env perl
0002
0003 use Switch;
0004 use POSIX "waitpid";
0005 use File::Basename;
0006
0007 $PYTHON_OPT = "--python_filename=l1t_test.py";
0008 $CMSRUN     = "cmsRun l1t_test.py";
0009
0010 $WORK_DIR = "test_l1t";
0011 $MAIN_LOG = "MAIN.log";
0012 $JOB_LOG = "JOB.log";
0013 $DIE_FILE = "DIE";
0014 $NUM_JOBS = 8;
0015 $TIMEOUT = 10*60;
0016 #$GEN_SIM_FILE = "/store/relval/CMSSW_7_6_0/RelValProdTTbar/GEN-SIM/76X_mcRun1_realistic_v10-v1/00000/08F44119-9C7F-E511-85C3-0025905B85A2.root";
0017 #$GEN_SIM_FILE = "file:/afs/cern.ch/work/m/mulhearn/data/gen-sim/GEN-SIM-08F44119-9C7F-E511-85C3-0025905B85A2.root";
0018 #$GEN_SIM_FILE = "root://cms-xrd-global.cern.ch///store/relval/CMSSW_7_6_0/RelValProdTTbar/GEN-SIM/76X_mcRun1_realistic_v10-v1/00000/08F44119-9C7F-E511-85C3-0025905B85A2.root";
0019 $GEN_SIM_FILE = "root://eoscms//eos/cms/store/group/dpg_trigger/comm_trigger/L1Trigger/rekovic/unit-test/input//store/relval/CMSSW_7_6_0/RelValProdTTbar/GEN-SIM/76X_mcRun1_realistic_v10-v1/00000/08F44119-9C7F-E511-85C3-0025905B85A2.root";
0020 #$DATA_2016_FILE = "root://eoscms//eos/cms/store/group/dpg_trigger/comm_trigger/L1Trigger/rekovic/unit-test/input//store/data/Run2016A/ZeroBias1/RAW/v1/000/271/336/00000/00963A5A-BF0A-E611-A657-02163E0141FB.root";
0021 $DATA_2016_FILE = "/store/data/Run2016H/HLTPhysics3/RAW/v1/000/281/602/00000/0E71643D-BF82-E611-8593-02163E011B19.root";
0022 #$DATA_2017_FILE = "/store/express/Run2017C/ExpressPhysics/FEVT/Express-v3/000/301/142/00000/0008C605-0881-E711-84A2-02163E019C07.root";
0023 #$DATA_2017_FILE = "/store/data/Run2017C/HLTPhysics/RAW/v1/000/301/142/00000/00682207-3181-E711-8706-02163E01445A.root";
0024 #$DATA_2017_FILE = "/store/data/Run2017G/HLTPhysics/RAW/v1/000/306/826/00000/3EFDFCB7-A2CE-E711-80E1-02163E019BEA.root";
0025 #$DATA_2017_FILE = "root://cms-xrd-global.cern.ch///store/data/Run2017H/SingleMuon/RAW/v1/000/307/082/00000/FE78107C-90D2-E711-84AB-02163E01A586.root";
0026 #$DATA_2018_FILE = "/store/data/Run2018A/HLTPhysics/RAW/v1/000/316/994/00000/CEFB7B46-3B62-E811-BA1F-FA163E4C0F07.root";
0027 #$DATA_2017_FILE = "root://cms-xrd-global.cern.ch///store/data/Run2017C/JetHT/RAW/v1/000/301/914/00000/C4B8EA5E-988A-E711-B38B-02163E01399E.root";
0028 $DATA_2017_FILE = "root://eoscms.cern.ch//eos/cms/store/group/dpg_trigger/comm_trigger/L1Trigger/rekovic/unit-test/input//store/data/Run2017C/JetHT/RAW/v1/000/301/914/00000/C4B8EA5E-988A-E711-B38B-02163E01399E.root";
0029 #$DATA_2017_FILE = "root://cms-xrd-global.cern.ch///store/express/Run2017C/ExpressPhysics/FEVT/Express-v2/000/300/226/00000/0218D0FB-2376-E711-B5B1-02163E014257.root";
0030 #$DATA_2017_FILE = "root://cms-xrd-global.cern.ch///store/data/Run2017D/HLTPhysics9/RAW/v1/000/302/676/00000/7A9DB54D-6D98-E711-B9AE-02163E013478.root";
0031 #$DATA_2018_FILE = "root://cms-xrd-global.cern.ch///store/data/Run2018A/SingleMuon/RAW/v1/000/316/218/00000/3CA8127E-4E57-E811-887E-FA163EEA1DC3.root";
0032 #$DATA_2018_FILE = "root://cms-xrd-global.cern.ch///store/express/Run2017F/ExpressPhysics/FEVT/Express-v1/000/305/636/00002/581DD5E7-10BA-E711-8E38-02163E011B94.root";
0033 #$DATA_2018_FILE = "/store/express/Run2018C/ExpressPhysics/FEVT/Express-v1/000/319/992/00000/041AB4A0-708C-E811-921F-02163E0153F7.root";
0034 #$DATA_2018_FILE = "/store/express/Run2018C/ExpressPhysics/FEVT/Express-v1/000/319/992/00000/041AB4A0-708C-E811-921F-02163E0153F7.root";
0035 #$DATA_2018_FILE = "root://cms-xrd-global.cern.ch///store/data/Run2018A/JetHT/RAW/v1/000/316/995/00000/FEEAE7BF-CA64-E811-8959-FA163E35C0CD.root";
0036 #$DATA_2018_FILE = "root://eoscms//eos/cms/store/group/dpg_trigger/comm_trigger/L1Trigger/rekovic/unit-test/input//data/Run2018A/JetHT/RAW/v1/000/316/995/00000/FEEAE7BF-CA64-E811-8959-FA163E35C0CD.root";
0037 $DATA_2018_FILE = "root://eoscms.cern.ch//eos/cms/store/group/dpg_trigger/comm_trigger/L1Trigger/rekovic/unit-test/input/store/data/Run2018B/JetHT/RAW/v1/000/316/995/00000/FEEAE7BF-CA64-E811-8959-FA163E35C0CD.root";
0038 #cmsDriver.py $PYTHON_OPT -s RAW2DIGI --era=Run2_2018 --data --conditions=101X_dataRun2_Prompt_v9 --customise=L1Trigger/Configuration/customiseReEmul.L1TReEmulFromRAW --customise=L1Trigger/L1TNtuples/customiseL1Ntuple.L1NtupleAODRAWEMU --customise=L1Trigger/Configuration/customiseSettings.L1TSettingsToCaloParams_2018_v1_2 --filein=/store/express/Run2017F/ExpressPhysics/FEVT/Express-v1/000/305/636/00002/581DD5E7-10BA-E711-8E38-02163E011B94.root
0039
0040 $VERBOSE  = 0;
0041 $KILL     = 0;
0042 $DRYRUN   = 0;
0043 $DELETE   = 0;
0044 $FAST     = 0;
0045 $SLOW     = 0;
0046 $RECYCLE  = 0;
0047 $VISUAL   = 0;
0048 $REDO     = 0;
0049 $REEMUL   = 0;
0050 $COMPARE  = 0;
0051 $COMPARE2  = 0;
0052 $COMPARE_DIR = "";
0053 $COMPARE_DIRS = "";
0054 $SINGLE  = 0;
0055 $SINGLE_JOB  = 0;
0056
0057 $COND_MC   = "--conditions=auto:run2_mc";
0058 #$COND_MC   = "--conditions=90X_mcRun2_asymptotic_v0";
0059 #$COND_MC   = "--conditions=90X_upgrade2017_realistic_v6_C1";
0060 #$COND_DATA_2015 = "--conditions=auto:run2_data";
0061 #$COND_DATA_2016 = "--conditions=auto:run2_data";
0062 #$COND_DATA_2017 = "--conditions=auto:run2_data";
0063 #$COND_DATA_2018 = "--conditions=auto:run2_data";
0064 $COND_DATA_2015 = "--conditions=101X_dataRun2_Prompt_v9";
0065 $COND_DATA_2016 = "--conditions=101X_dataRun2_Prompt_v9";
0066 $COND_DATA_2017 = "--conditions=101X_dataRun2_Prompt_v9";
0067 $COND_DATA_2018 = "--conditions=101X_dataRun2_Prompt_v9";
0068 #$COND_DATA_2017 = "--conditions=92X_dataRun2_Prompt_v8";
0069 #$COND_MC   = "--conditions=80X_mcRun2_asymptotic_v17";  # = run2_mc in CMSSW_8_0_19
0070 #$COND_DATA_2015 = "--conditions=80X_dataRun2_v18";      # = run2_data in CMSSW_8_0_19
0071 #$COND_DATA_2016 = "--conditions=80X_dataRun2_Prompt_v8";
0072
0073 sub main;
0074 main @ARGV;
0075
0076 sub usage() {
0077     print "usage: testL1T.pl [opt]\n";
0078     print "\n";
0079     print "Integration test for L1T.\n";
0080     print "\n";
0081     print "Possible options:\n";
0082     print "--help             display this message.\n";
0083     print "--kill             cleanly kill a running instance of testL1T.pl.\n";
0084     print "--verbose          output lots of information.\n";
0085     print "--delete           delete previous job directory if it exists.\n";
0086     print "--fast             limit the number of events for an initial quick test.\n";
0087     print "--slow             increase the number of events for a slower but more thorough test.\n";
0088     print "--dryrun           don't launch any long jobs, just show what would be done.\n";
0089     print "--recycle          recycle previous results, re-running evaluation only (for debugging).\n";
0090     print "--visual           run some quick, visual checks.\n";
0091     print "--reemul           generate command-line supported reEmul.py script and exit\n";
0092     print "--compare=<d>      compare ntuples produced here with those in directory <d>\n";
0093     print "--compare2=<d1:d2> compare ntuples produced in directory <d1> with those in directory <d2>\n";
0094     print "--single=<d>       only run single job <d>\n";
0095     print "--redo             redo any failed jobs\n";
0096     exit 0;
0097 }
0098
0099 $CHILD_PID = 0;
0100 sub long_command {
0101     my $cmd = shift;
0102     if ($DRYRUN){
0103     print "INFO: due to --dryrun, not running command:  $cmd\n";
0104     return 0;
0105     }
0106     print "INFO: running command:  $cmd\n";
0107     $CHILD_PID = fork() or exec($cmd);
0108     waitpid( -1, WNOHANG );
0109     $CHILD_PID = 0;
0110     return $?;
0111 }
0112
0113 #
0114 # These are simple visual checks...
0115 #
0116 sub visual_sim_pack_unpack {
0117     $status = long_command("cmsDriver.py L1TEST $PYTHON_OPT $COND_MC -s DIGI:pdigi_valid,L1,DIGI2RAW,RAW2DIGI -n 5 --era Run2_2016 --filein=$GEN_SIM_FILE --mc --no_output --no_exec  --customise=L1Trigger/Configuration/customiseUtils.L1TStage2SimDigisSummary --customise=L1Trigger/Configuration/customiseUtils.L1TStage2DigisSummary --customise=L1Trigger/Configuration/customiseUtils.L1TGlobalSimDigisSummary --customise=L1Trigger/Configuration/customiseUtils.L1TAddInfoOutput");
0118
0119     print "INFO: status of cmsDriver call is $status\n";
0120     if ($status){
0121     print "ERROR: abnormal status returned: $status\n";
0122     return;
0123     }
0124
0125     $status = long_command("$CMSRUN");
0126     print "INFO: status of cmsRun call is $status\n";
0127     if ($status){
0128     print "ERROR: abnormal status returned: $status\n";
0129     return;
0130     }
0131     print "INFO: visual test has finished without abnormal status...\n";
0132 }
0133
0134 sub visual_unpack {
0135     $file = "/store/data/Commissioning2016/Cosmics/RAW/v1/000/264/573/00000/5A9E5261-BDD1-E511-9102-02163E014378.root";
0136     $status = long_command("cmsDriver.py L1TEST $PYTHON_OPT -s RAW2DIGI --era=Run2_2016 $COND_DATA_2015 -n 100 --data --filein=$file --no_output --no_exec --customise=L1Trigger/Configuration/customiseUtils.L1TStage2DigisSummary --customise=L1Trigger/Configuration/customiseUtils.L1TGlobalDigisSummary --customise=L1Trigger/Configuration/customiseUtils.L1TAddInfoOutput --customise=L1Trigger/Configuration/customiseUtils.L1TGlobalMenuXML");
0137
0138     print "INFO: status of cmsDriver call is $status\n";
0139     if ($status){
0140     print "ERROR: abnormal status returned: $status\n";
0141     return;
0142     }
0143     $status = long_command("$CMSRUN");
0144     print "INFO: status of cmsRun call is $status\n";
0145     if ($status){
0146     print "ERROR: abnormal status returned: $status\n";
0147     return;
0148     }
0149     print "INFO: visual test has finished without abnormal status...\n";
0150 }
0151
0152
0153
0154 #
0155 # This is a dummy test:
0156 #
0157 sub test_dummy {
0158     # for process cleanup to work, make system calls to long processes like this:
0159     if (! $RECYCLE) {
0160     long_command("sleep 20");
0161     }
0162     system "touch SUCCESS";
0163 }
0164
0165 #
0166 # a simple check that unpackers do not crash on recent RAW data
0167 #
0168 sub test_unpackers_dont_crash {
0169     #$file = "/store/data/Commissioning2016/Cosmics/RAW/v1/000/264/573/00000/5A9E5261-BDD1-E511-9102-02163E014378.root";
0170     #$file = "/store/data/Run2016A/ZeroBias1/RAW/v1/000/271/336/00000/00963A5A-BF0A-E611-A657-02163E0141FB.root";
0171     $nevt = 200;
0172     if ($FAST) {$nevt = 10; }
0173     if ($SLOW) {$nevt = -1; }
0174     if (! $RECYCLE){
0175     $status = long_command("cmsDriver.py L1TEST $PYTHON_OPT -s RAW2DIGI --era=Run2_2016 $COND_DATA_2015 -n $nevt --data --filein=$DATA_2016_FILE --no_output  --no_exec >& CMSDRIVER.log");
0176     print "INFO: status of cmsDriver call is $status\n";
0177     if ($status){
0178         print "ERROR: abnormal status returned: $status\n";
0179         return;
0180     }
0181     $status = long_command("$CMSRUN >& CMSRUN.log");
0182     print "INFO: status of cmsRun call is $status\n";
0183     if ($status){
0184         print "ERROR: abnormal status returned: $status\n";
0185         return;
0186     }
0187     }
0188     system "touch SUCCESS";
0189 }
0190
0191
0192 #
0193 # Test unpack non-zero payloads from 2016 Data
0194 #
0195 sub test_unpack_2016_data {
0196     # this one runs a bit slower so scale number of events:
0197     $nevt = 200;
0198     if ($FAST) {$nevt = 100; }
0199     if ($SLOW) {$nevt = 500; }
0200
0201     if (! $RECYCLE){
0202     $status = long_command("cmsDriver.py L1TEST $PYTHON_OPT $COND_DATA_2016 -s RAW2DIGI -n $nevt --era=Run2_2016 --data --filein=$DATA_2016_FILE --no_output --no_exec --customise=L1Trigger/Configuration/customiseUtils.L1TStage2DigisSummary --customise=L1Trigger/L1TNtuples/customiseL1Ntuple.L1NtupleRAW >& CMSDRIVER.log");
0203
0204     print "INFO: status of cmsDriver call is $status\n";
0205     if ($status){
0206         print "ERROR: abnormal status returned: $status\n";
0207         return;
0208     }
0209     $status = long_command("$CMSRUN >& CMSRUN.log");
0210     print "INFO: status of cmsRun call is $status\n";
0211     if ($status){
0212         print "ERROR: abnormal status returned: $status\n";
0213         return;
0214     }
0215     }
0216     open INPUT,"root -b -q -x '../../L1Trigger/L1TCommon/macros/CheckL1Ntuple.C(\"L1Ntuple.root\",\"l1UpgradeTree/L1UpgradeTree\")' |";
0217     while (<INPUT>){
0218     print $_;
0219     if (/SUCCESS/){
0220         system "touch SUCCESS";
0221     }
0222     }
0223 }
0224
0225
0226
0227 #
0228 # check that pack unpack is unity
0229 #
0230 sub test_pack_unpack_is_unity {
0231     # this one runs a bit slower so scale number of events:
0232     $nevt = 50;
0233     if ($FAST) {$nevt = 5; }
0234     if ($SLOW) {$nevt = 500; }
0235
0236     if (! $RECYCLE){
0237     $status = long_command("cmsDriver.py L1TEST $PYTHON_OPT $COND_MC -s DIGI,L1,DIGI2RAW,RAW2DIGI -n $nevt --era Run2_2016 --mc --no_output --no_exec --filein=$GEN_SIM_FILE --customise=L1Trigger/Configuration/customiseUtils.L1TStage2ComparisonRAWvsEMU --customise=L1Trigger/Configuration/customiseUtils.L1TGtStage2ComparisonRAWvsEMU >& CMSDRIVER.log");
0238 # --geometry=Extended2016,Extended2016Reco --customise=L1Trigger/Configuration/customiseReEmul.L1TEventSetupForHF1x1TPs
0239
0240     print "INFO: status of cmsDriver call is $status\n";
0241     if ($status){
0242         print "ERROR: abnormal status returned: $status\n";
0243         return;
0244     }
0245     $status = long_command("$CMSRUN >& CMSRUN.log");
0246     print "INFO: status of cmsRun call is $status\n";
0247     if ($status){
0248         print "ERROR: abnormal status returned: $status\n";
0249         return;
0250     }
0251     }
0252     $count = 0;
0253     open (INPUT, "grep \"SUMMARY:  L1T Comparison\" CMSRUN.log |");
0254     while (<INPUT>){
0255     if (/SUCCESS/) {$count++;}
0256     }
0257     if ($count == 2) {system "touch SUCCESS"; }
0258 }
0259
0260
0261
0262 #
0263 # Test the re-emulation sequence:
0264 #
0265 # - checks cmsDriver.py command for Stage-2 full re-emulation from RAW, saving re-emulation output to ntuple
0266 # - checks cmsRun on resultant config
0267 # - checks ntuple for non-zero number of jets, taus, e-gammas, and muons.
0268 #
0269 sub test_reemul {
0270 #    $file = "/store/data/Run2015D/DoubleEG/RAW-RECO/ZElectron-PromptReco-v4/000/260/627/00000/12455212-1E85-E511-8913-02163E014472.root";
0271     $file = "/store/data/Run2015D/MuonEG/RAW/v1/000/256/677/00000/4A874FB5-585D-E511-A3D8-02163E0143B5.root";
0272     $nevt = 100;
0273     if ($FAST) {$nevt = 10; }
0274     if ($SLOW) {$nevt = 1000; }
0275
0276     if (! $RECYCLE){
0277     $status = long_command("cmsDriver.py L1TEST $PYTHON_OPT -s RAW2DIGI --era=Run2_2016 --customise=L1Trigger/Configuration/customiseReEmul.L1TReEmulFromRAW2015 --customise=L1Trigger/L1TNtuples/customiseL1Ntuple.L1NtupleEMU $COND_DATA_2015 -n $nevt --data --no_exec --no_output --filein=$file --geometry=Extended2016,Extended2016Reco --customise=L1Trigger/Configuration/customiseReEmul.L1TEventSetupForHF1x1TPs --customise=L1Trigger/Configuration/customiseUtils.L1TGlobalSimDigisSummary --customise=L1Trigger/Configuration/customiseUtils.L1TAddInfoOutput >& CMSDRIVER.log");
0278
0279     print "INFO: status of cmsDriver call is $status\n";
0280     if ($status){
0281         print "ERROR: abnormal status returned: $status\n";
0282         return;
0283     }
0284     $status = long_command("$CMSRUN >& CMSRUN.log");
0285     print "INFO: status of cmsRun call is $status\n";
0286     if ($status){
0287         print "ERROR: abnormal status returned: $status\n";
0288         return;
0289     }
0290     }
0291
0292     $SUCCESS = 0;
0293     open INPUT,"root -b -q -x ../../L1Trigger/L1TCommon/macros/CheckL1Ntuple.C |";
0294     while (<INPUT>){
0295     print $_;
0296     if (/SUCCESS/){ $SUCCESS = 1; }
0297     }
0298     close INPUT;
0299
0300     if (! $SUCCESS){
0301     print "ERROR:  L1Ntuple did not contain sufficient Calo and Muon candidates for success.\n";
0302     return;
0303     }
0304
0305     #print "INFO: parsing the following menu summary:\n";
0306     #system "grep 'L1T menu Name' -A 250 CMSRUN.log";
0307
0308     @TRIGGERS = ("L1_SingleMu5","L1_SingleEG5","L1_SingleJet52");
0309     foreach $trig (@TRIGGERS) {
0310     open INPUT,"grep 'L1T menu Name' -A 250 CMSRUN.log | grep $trig |";
0311     $FIRED = 0;
0312     while (<INPUT>){
0313         #chomp; print "LINE:  $_\n";
0314         /$trig\W+(\w+)/;
0315         print "INFO:  $trig fired $1 times\n";
0316         if ($1 > 0){ $FIRED = 1; }
0317     }
0318     if (! $FIRED){
0319         print "ERROR:  $trig did not fire.\n";
0320         return;
0321     }
0322     }
0323     system "touch SUCCESS";
0324
0325 }
0326
0327 sub test_reemul_2016 {
0328 #    $file = "/store/data/Run2015D/DoubleEG/RAW-RECO/ZElectron-PromptReco-v4/000/260/627/00000/12455212-1E85-E511-8913-02163E014472.root";
0329     #$file = "/store/data/Run2016A/ZeroBias1/RAW/v1/000/271/336/00000/00963A5A-BF0A-E611-A657-02163E0141FB.root";
0330     #$file = "/store/data/Run2016B/MuonEG/RAW-RECO/TopMuEG-PromptReco-v2/000/273/450/00000/8EF765B2-531C-E611-86D5-02163E012060.root";
0331     #$file = "/store/data/Run2016B/ZeroBias8/RAW/v1/000/272/021/00000/D821E02F-BA0D-E611-9DE0-02163E013885.root";
0332     $nevt = 100;
0333     if ($FAST) {$nevt = 200; }
0334     if ($SLOW) {$nevt = 1000; }
0335
0336     if (! $RECYCLE){
0337     $status = long_command("cmsDriver.py $PYTHON_OPT -s RAW2DIGI --era=Run2_2016 --customise=L1Trigger/Configuration/customiseReEmul.L1TReEmulFromRAW --customise=L1Trigger/L1TNtuples/customiseL1Ntuple.L1NtupleEMU --customise=L1Trigger/Configuration/customiseUtils.L1TTurnOffUnpackStage2GtGmtAndCalo $COND_DATA_2015 -n $nevt --data --no_exec --no_output --filein=$DATA_2016_FILE --customise=L1Trigger/Configuration/customiseUtils.L1TGlobalSimDigisSummary --customise=L1Trigger/Configuration/customiseUtils.L1TAddInfoOutput --customise=L1Trigger/Configuration/customiseUtils.L1TGlobalMenuXML >& CMSDRIVER.log");
0338
0339     print "INFO: status of cmsDriver call is $status\n";
0340     if ($status){
0341         print "ERROR: abnormal status returned: $status\n";
0342         return;
0343     }
0344     $status = long_command("$CMSRUN >& CMSRUN.log");
0345     print "INFO: status of cmsRun call is $status\n";
0346     if ($status){
0347         print "ERROR: abnormal status returned: $status\n";
0348         return;
0349     }
0350     }
0351
0352     $SUCCESS = 0;
0353     open INPUT,"root -b -q -x ../../L1Trigger/L1TCommon/macros/CheckL1Ntuple.C |";
0354     while (<INPUT>){
0355     print $_;
0356     if (/SUCCESS/){ $SUCCESS = 1; }
0357     }
0358     close INPUT;
0359
0360     if (! $SUCCESS){
0361     print "ERROR:  L1Ntuple did not contain sufficient Calo and Muon candidates for success.\n";
0362     return;
0363     }
0364
0365     #print "INFO: parsing the following menu summary:\n";
0366     #system "grep 'L1T menu Name' -A 250 CMSRUN.log";
0367
0368     @TRIGGERS = ("L1_SingleMu3","L1_SingleEG5","L1_SingleJet16");
0369     foreach $trig (@TRIGGERS) {
0370     open INPUT,"grep 'L1T menu Name' -A 250 CMSRUN.log | grep $trig |";
0371     $FIRED = 0;
0372     while (<INPUT>){
0373         #chomp; print "LINE:  $_\n";
0374         /$trig\W+(\w+)/;
0375         print "INFO:  $trig fired $1 times\n";
0376         if ($1 > 0){ $FIRED = 1; }
0377     }
0378     if (! $FIRED){
0379         print "ERROR:  $trig did not fire.\n";
0380         return;
0381     }
0382     }
0383     system "touch SUCCESS";
0384
0385 }
0386
0387 sub test_reemul_2017 {
0388     $nevt = 100;
0389     if ($FAST) {$nevt = 100; }
0390     if ($SLOW) {$nevt = 1000; }
0391
0392     if (! $RECYCLE){
0393     $status = long_command("cmsDriver.py L1TEST $PYTHON_OPT -s RAW2DIGI --era=Run2_2017 --customise=L1Trigger/Configuration/customiseReEmul.L1TReEmulFromRAW --customise=L1Trigger/L1TNtuples/customiseL1Ntuple.L1NtupleEMU --customise=L1Trigger/Configuration/customiseUtils.L1TTurnOffUnpackStage2GtGmtAndCalo $COND_DATA_2017 -n $nevt --data --no_exec --no_output --filein=$DATA_2017_FILE --customise=L1Trigger/Configuration/customiseUtils.L1TGlobalSimDigisSummary --customise=L1Trigger/Configuration/customiseUtils.L1TAddInfoOutput >& CMSDRIVER.log");
0394
0395     print "INFO: status of cmsDriver call is $status\n";
0396     if ($status){
0397         print "ERROR: abnormal status returned: $status\n";
0398         return;
0399     }
0400     $status = long_command("$CMSRUN >& CMSRUN.log");
0401     print "INFO: status of cmsRun call is $status\n";
0402     if ($status){
0403         print "ERROR: abnormal status returned: $status\n";
0404         return;
0405     }
0406     }
0407
0408     $SUCCESS = 0;
0409     open INPUT,"root -b -q -x ../../L1Trigger/L1TCommon/macros/CheckL1Ntuple.C |";
0410     while (<INPUT>){
0411     print $_;
0412     if (/SUCCESS/){ $SUCCESS = 1; }
0413     }
0414     close INPUT;
0415
0416     if (! $SUCCESS){
0417     print "ERROR:  L1Ntuple did not contain sufficient Calo and Muon candidates for success.\n";
0418     return;
0419     }
0420
0421     #print "INFO: parsing the following menu summary:\n";
0422     #system "grep 'L1T menu Name' -A 250 CMSRUN.log";
0423
0424     @TRIGGERS = ("L1_SingleMu3","L1_SingleEG5","L1_SingleJet16");
0425     foreach $trig (@TRIGGERS) {
0426     open INPUT,"grep 'L1T menu Name' -A 250 CMSRUN.log | grep $trig |";
0427     $FIRED = 0;
0428     while (<INPUT>){
0429         #chomp; print "LINE:  $_\n";
0430         /$trig\W+(\w+)/;
0431         print "INFO:  $trig fired $1 times\n";
0432         if ($1 > 0){ $FIRED = 1; }
0433     }
0434     if (! $FIRED){
0435         print "ERROR:  $trig did not fire.\n";
0436         return;
0437     }
0438     }
0439     system "touch SUCCESS";
0440
0441 }
0442
0443 sub test_reemul_2018 {
0444     #$file = "/store/express/Run2017C/ExpressPhysics/FEVT/Express-v3/000/301/142/00000/0008C605-0881-E711-84A2-02163E019C07.root"
0445     $nevt = 100;
0446     if ($FAST) {$nevt = 100; }
0447     if ($SLOW) {$nevt = 1000; }
0448
0449     if (! $RECYCLE){
0450     $status = long_command("cmsDriver.py L1TEST $PYTHON_OPT -s RAW2DIGI --era=Run2_2018 --customise=L1Trigger/Configuration/customiseReEmul.L1TReEmulFromRAW --customise=L1Trigger/L1TNtuples/customiseL1Ntuple.L1NtupleEMU --customise=L1Trigger/Configuration/customiseUtils.L1TTurnOffUnpackStage2GtGmtAndCalo $COND_DATA_2018 -n $nevt --data --no_exec --no_output --filein=$DATA_2018_FILE --customise=L1Trigger/Configuration/customiseUtils.L1TGlobalSimDigisSummary --customise=L1Trigger/Configuration/customiseUtils.L1TAddInfoOutput >& CMSDRIVER.log");
0451
0452     print "INFO: status of cmsDriver call is $status\n";
0453     if ($status){
0454         print "ERROR: abnormal status returned: $status\n";
0455         return;
0456     }
0457     $status = long_command("$CMSRUN >& CMSRUN.log");
0458     print "INFO: status of cmsRun call is $status\n";
0459     if ($status){
0460         print "ERROR: abnormal status returned: $status\n";
0461         return;
0462     }
0463     }
0464
0465     $SUCCESS = 0;
0466     open INPUT,"root -b -q -x ../../L1Trigger/L1TCommon/macros/CheckL1Ntuple.C |";
0467     while (<INPUT>){
0468     print $_;
0469     if (/SUCCESS/){ $SUCCESS = 1; }
0470     }
0471     close INPUT;
0472
0473     if (! $SUCCESS){
0474     print "ERROR:  L1Ntuple did not contain sufficient Calo and Muon candidates for success.\n";
0475     return;
0476     }
0477
0478     #print "INFO: parsing the following menu summary:\n";
0479     #system "grep 'L1T menu Name' -A 250 CMSRUN.log";
0480
0481     @TRIGGERS = ("L1_SingleMu3","L1_SingleEG50","L1_SingleJet35");
0482     foreach $trig (@TRIGGERS) {
0483     open INPUT,"grep 'L1T menu Name' -A 250 CMSRUN.log | grep $trig |";
0484     $FIRED = 0;
0485     while (<INPUT>){
0486         #chomp; print "LINE:  $_\n";
0487         /$trig\W+(\w+)/;
0488         print "INFO:  $trig fired $1 times\n";
0489         if ($1 > 0){ $FIRED = 1; }
0490     }
0491     if (! $FIRED){
0492         print "ERROR:  $trig did not fire.\n";
0493         return;
0494     }
0495     }
0496     system "touch SUCCESS";
0497
0498 }
0499
0500 sub test_mc_prod {
0501     $nevt = 50;
0502     if ($FAST) {$nevt = 5; }
0503     if ($SLOW) {$nevt = 500; }
0504
0505     if (! $RECYCLE){
0506     $status = long_command("cmsDriver.py L1TEST $COND_MC -s DIGI,L1 --datatier GEN-SIM-RAW -n $nevt --era Run2_2016 --mc --no_output --no_exec --filein=$GEN_SIM_FILE --customise=L1Trigger/L1TNtuples/customiseL1Ntuple.L1NtupleEMUNoEventTree  --customise=L1Trigger/Configuration/customiseUtils.L1TGlobalSimDigisSummary --customise=L1Trigger/Configuration/customiseUtils.L1TAddInfoOutput >& CMSDRIVER.log");
0507 # --geometry=Extended2016,Extended2016Reco --customise=L1Trigger/Configuration/customiseReEmul.L1TEventSetupForHF1x1TPs
0508
0509     print "INFO: status of cmsDriver call is $status\n";
0510     if ($status){
0511         print "ERROR: abnormal status returned: $status\n";
0512         return;
0513     }
0514     $status = long_command("cmsRun L1TEST_DIGI_L1.py >& CMSRUN.log");
0515     print "INFO: status of cmsRun call is $status\n";
0516     if ($status){
0517         print "ERROR: abnormal status returned: $status\n";
0518         return;
0519     }
0520     }
0521     open INPUT,"root -b -q -x ../../L1Trigger/L1TCommon/macros/CheckL1Ntuple.C |";
0522     while (<INPUT>){
0523     print $_;
0524     if (/SUCCESS/){
0525         system "touch SUCCESS";
0526     }
0527     }
0528 }
0529
0530
0531
0532
0533 sub run_job {
0534     my $ijob = shift;
0535     $SIG{HUP} = sub {
0536     print "ERROR: job $ijob received HUP.. exiting\n";
0537     if ($CHILD_PID) { kill HUP => $CHILD_PID; }
0538     exit 0;
0539     };
0540     $JOBDIR = "test_$ijob";
0541     if ($RECYCLE){
0542     if (! -e $JOBDIR){
0543         print "ERROR:  --recycle specified but $JOBDIR does not exist!\n";
0544         return;
0545     }
0546     chdir $JOBDIR;
0547     if (-e "SUCCESS"){ system "rm SUCCESS"; }
0548     } else {
0549     system "mkdir $JOBDIR";
0550     chdir $JOBDIR;
0551     }
0552     open STDOUT,">",$JOB_LOG or die $!;
0553     open STDERR,">",$JOB_LOG or die $!;
0554     print "INFO: job $ijob starting...\n";
0555     my $start_time = time();
0556     switch ($ijob){
0557     #case 0 {test_dummy; }
0558     case 0 {test_reemul;}
0559     case 1 {test_mc_prod; }
0560     case 2 {test_unpack_2016_data; }
0561     case 3 {test_pack_unpack_is_unity; }
0562     case 4 {test_unpackers_dont_crash; }
0563     case 5 {test_reemul_2016;}
0564     case 6 {test_reemul_2017;}
0565     case 7 {test_reemul_2018;}
0566     else   {test_dummy; }
0567     }
0568     my $job_time = time() - $start_time;
0569     print "INFO: job $ijob ending after $job_time seconds...\n";
0570     exit;
0571 }
0572
0573
0574 sub main {
0575     my @args = ();
0576     # parse the command line arguments:
0577     my $arg;
0578     while($arg = shift){
0579         if ($arg =~ /^--/){
0580             if ($arg =~ /--help/)      { usage();                 }
0581             elsif ($arg =~ /--verbose/)   { $VERBOSE   = 1;          }
0582             elsif ($arg =~ /--kill/)      { $KILL      = 1;          }
0583             elsif ($arg =~ /--dryrun/)    { $DRYRUN    = 1;          }
0584             elsif ($arg =~ /--delete/)    { $DELETE    = 1;          }
0585             elsif ($arg =~ /--fast/)      { $FAST      = 1;          }
0586             elsif ($arg =~ /--slow/)      { $SLOW      = 1;          }
0587             elsif ($arg =~ /--recycle/)   { $RECYCLE   = 1;          }
0588             elsif ($arg =~ /--visual/)    { $VISUAL    = 1;          }
0589             elsif ($arg =~ /--reemul/)    { $REEMUL    = 1;          }
0590             elsif ($arg =~ /--compare=(\S+)/) { $COMPARE = 1; $COMPARE_DIR=$1;}
0591             elsif ($arg =~ /--compare2=(\S+)/) { $COMPARE2 = 1; $COMPARE_DIRS=$1;}
0592             elsif ($arg =~ /--single=(\S+)/) { $SINGLE = 1; $SINGLE_JOB=$1;}
0593             elsif ($arg =~ /--redo/)      { $REDO      = 1;          }
0594         else {print "ERROR: unrecognized argument: $arg\n"; usage(); }
0595         } else {
0596             push @args, $arg;
0597         }
0598     }
0599     if ($#args != -1){ usage(); }
0600
0601     if ($FAST && $SLOW){ usage(); }
0602     if ($FAST){ $WORK_DIR = "${WORK_DIR}_fast"; $TIMEOUT = 5*60;}
0603     if ($SLOW){ $WORK_DIR = "${WORK_DIR}_slow"; $TIMEOUT = 30*60;}
0604
0605
0606     if ($KILL) {
0607     print "INFO: killing testL1T.pl instance running at $WORK_DIR\n";
0608     system "touch $WORK_DIR/$DIE_FILE";
0609     system "sleep 3";
0610     if (-e "$WORK_DIR/$DIE_FILE"){
0611         print "ERROR: kill was not successfull.. perhaps that instance already finished?\n";
0612         system "rm $WORK_DIR/$DIE_FILE";
0613     }
0614     exit(0);
0615     }
0616
0617     print "INFO: Welcome the L1T offline software integration testing!\n";
0618
0619     if ($SINGLE){
0620     print "INFO:  will run only single job $SINGLE_JOB\n";
0621     $WORK_DIR = "${WORK_DIR}_single_${SINGLE_JOB}";
0622     }
0623     if ($REDO){
0624     print "INFO:  will redo all failed jobs.\n";
0625     }
0626
0627     if ($REEMUL){
0628     $nevt = 10;
0629     $file = "/store/data/Run2015D/MuonEG/RAW/v1/000/256/677/00000/4A874FB5-585D-E511-A3D8-02163E0143B5.root";
0630     $status = long_command("cmsDriver.py L1TEST --python_filename=reEmul.py -s RAW2DIGI --era=Run2_2016 --customise=L1Trigger/Configuration/customiseReEmul.L1TReEmulFromRAW2015 --customise=L1Trigger/L1TNtuples/customiseL1Ntuple.L1NtupleEMU --customise=L1Trigger/Configuration/customiseUtils.L1TTurnOffUnpackStage2GtGmtAndCalo $COND_DATA_2015 -n $nevt --data --no_exec --no_output --filein=$file --geometry=Extended2016,Extended2016Reco --customise=L1Trigger/Configuration/customiseReEmul.L1TEventSetupForHF1x1TPs >& CMSDRIVER.log");
0631     if ($status){
0632         print "ERROR: abnormal status returned: $status\n";
0633         return;
0634     }
0635     print "INFO: adding command line option support\n";
0636         system "cat L1Trigger/L1TCommon/scripts/optionsL1T.py >> reEmul.py";
0637     exit(0);
0638     }
0639     if ($COMPARE){
0640     print "INFO: Comparing results with those in $COMPARE_DIR\n";
0641     $ours = "$WORK_DIR/test_0/L1Ntuple.root";
0642     $theirs = "$COMPARE_DIR/$WORK_DIR/test_0/L1Ntuple.root";
0643     if (! -e $ours)   { print "ERROR: could not find file $ours\n"; exit(1); }
0644     if (! -e $theirs) { print "ERROR: could not find file $theirs\n"; exit(1); }
0645     print "$ours\n";
0646     print "$theirs\n";;
0647     $status = long_command("root -b -q -x 'L1Trigger/L1TCommon/macros/NtupleDiff.C(\"reemul\",\"$ours\",\"$theirs\")'");
0648
0649     $ours = "$WORK_DIR/test_1/L1Ntuple.root";
0650     $theirs = "$COMPARE_DIR/$WORK_DIR/test_1/L1Ntuple.root";
0651     if (! -e $ours)   { print "ERROR: could not find file $ours\n"; exit(1); }
0652     if (! -e $theirs) { print "ERROR: could not find file $theirs\n"; exit(1); }
0653     print "$ours\n";
0654     print "$theirs\n";;
0655     $status = long_command("root -b -q -x 'L1Trigger/L1TCommon/macros/NtupleDiff.C(\"mc\",\"$ours\",\"$theirs\")'");
0656
0657     # this is a hack until L1T uGT output goes into L1TNtuple:
0658         system "sed -n \'/L1T menu Name/,/Final OR Count/p\' $WORK_DIR/test_0/CMSRUN.log > menu_a.txt";
0659         system "sed -n \'/L1T menu Name/,/Final OR Count/p\' $COMPARE_DIR/$WORK_DIR/test_0/CMSRUN.log > menu_b.txt";
0660     print "INFO:  diff of menu summary follows:\n";
0661     system "diff menu_a.txt menu_b.txt\n";
0662     exit(0);
0663     }
0664
0665     if ($COMPARE2){
0666
0667         if ($FAST) {$nevt = 5; }
0668         if ($SLOW) {$nevt = 500; }
0669
0670 #my $CUR_DIR = cwd();
0671
0672         my ($DIR1,$DIR2) = split /:/, $COMPARE_DIRS;
0673     print "INFO: Comparing results in $DIR1 with those in $DIR2\n";
0674
0675         # make comparision dir
0676         my $compStr1 = basename ($DIR1);
0677         my $compStr2 = basename ($DIR2);
0678         $COMP_DIR = "compare_$compStr1\_vs_$compStr2";
0679         mkdir($COMP_DIR) unless(-d $COMP_DIR);
0680
0681         # go to comparision dir
0682     chdir $COMP_DIR;
0683     system "pwd";
0684
0685     $ours = "../$DIR1/$WORK_DIR/test_0/L1Ntuple.root";
0686     $theirs = "../$DIR2/$WORK_DIR/test_0/L1Ntuple.root";
0687     if (! -e $ours)   { print "ERROR: could not find file $ours\n"; exit(1); }
0688     if (! -e $theirs) { print "ERROR: could not find file $theirs\n"; exit(1); }
0689     print "$ours\n";
0690     print "$theirs\n";;
0691     $status = long_command("root -b -q -x '$ENV{CMSSW_BASE}/src/L1Trigger/L1TCommon/macros/NtupleDiff.C(\"reemul\",\"$ours\",\"$theirs\")'");
0692
0693     $ours = "../$DIR1/$WORK_DIR/test_1/L1Ntuple.root";
0694     $theirs = "../$DIR2/$WORK_DIR/test_1/L1Ntuple.root";
0695     if (! -e $ours)   { print "ERROR: could not find file $ours\n"; exit(1); }
0696     if (! -e $theirs) { print "ERROR: could not find file $theirs\n"; exit(1); }
0697     print "$ours\n";
0698     print "$theirs\n";;
0699     $status = long_command("root -b -q -x '$ENV{CMSSW_BASE}/src/L1Trigger/L1TCommon/macros/NtupleDiff.C(\"mc\",\"$ours\",\"$theirs\")'");
0700
0701     $ours = "../$DIR1/$WORK_DIR/test_2/L1Ntuple.root";
0702     $theirs = "../$DIR2/$WORK_DIR/test_2/L1Ntuple.root";
0703     if (! -e $ours)   { print "ERROR: could not find file $ours\n"; exit(1); }
0704     if (! -e $theirs) { print "ERROR: could not find file $theirs\n"; exit(1); }
0705     print "$ours\n";
0706     print "$theirs\n";;
0707     $status = long_command("root -b -q -x '$ENV{CMSSW_BASE}/src/L1Trigger/L1TCommon/macros/NtupleDiff.C(\"raw2016\",\"$ours\",\"$theirs\",\"l1UpgradeTree/L1UpgradeTree\",\"l1UpgradeTree/L1UpgradeTree\")'");
0708
0709     $ours = "../$DIR1/$WORK_DIR/test_5/L1Ntuple.root";
0710     $theirs = "../$DIR2/$WORK_DIR/test_5/L1Ntuple.root";
0711     if (! -e $ours)   { print "ERROR: could not find file $ours\n"; exit(1); }
0712     if (! -e $theirs) { print "ERROR: could not find file $theirs\n"; exit(1); }
0713     print "$ours\n";
0714     print "$theirs\n";;
0715     $status = long_command("root -b -q -x '$ENV{CMSSW_BASE}/src/L1Trigger/L1TCommon/macros/NtupleDiff.C(\"reemul2016\",\"$ours\",\"$theirs\")'");
0716
0717     $ours = "../$DIR1/$WORK_DIR/test_6/L1Ntuple.root";
0718     $theirs = "../$DIR2/$WORK_DIR/test_6/L1Ntuple.root";
0719     if (! -e $ours)   { print "ERROR: could not find file $ours\n"; exit(1); }
0720     if (! -e $theirs) { print "ERROR: could not find file $theirs\n"; exit(1); }
0721     print "$ours\n";
0722     print "$theirs\n";;
0723     $status = long_command("root -b -q -x '$ENV{CMSSW_BASE}/src/L1Trigger/L1TCommon/macros/NtupleDiff.C(\"reemul2017\",\"$ours\",\"$theirs\")'");
0724
0725
0726     # this is a hack until L1T uGT output goes into L1TNtuple:
0727         system "sed -n \'/L1T menu Name/,/Final OR Count/p\' ../$DIR1/$WORK_DIR/test_0/CMSRUN.log > menu_a.txt";
0728         system "sed -n \'/L1T menu Name/,/Final OR Count/p\' ../$DIR2/$WORK_DIR/test_0/CMSRUN.log > menu_b.txt";
0729     print "INFO:  diff of menu summary of $compStr1 vs $compStr2 follows:\n";
0730     system "echo \'diff l1t menu with $nevt events of MC\' > diff_menu_a_vs_menu_b.txt\n";
0731     system "echo \'< $compStr1\' >> diff_menu_a_vs_menu_b.txt\n";
0732     system "echo \'> $compStr2\' >> diff_menu_a_vs_menu_b.txt\n";
0733     system "echo \'---------------------------------------------------------' >> diff_menu_a_vs_menu_b.txt\n";
0734     system "sleep 1";
0735     system "diff menu_a.txt menu_b.txt >> diff_menu_a_vs_menu_b.txt\n";
0736     $status = long_command("bash $ENV{CMSSW_BASE}/src/L1Trigger/L1TCommon/scripts/makeHtml.sh $compStr1 $compStr2");
0737
0738         # go back to original dir
0739     chdir "..";
0740     exit(0);
0741     }
0742
0743
0744
0745     if ($VISUAL){
0746     print "INFO: visual mode was specified... note that successful results do not green-light a commit.\n";
0747     visual_unpack();
0748     visual_sim_pack_unpack();
0749     exit(0);
0750     }
0751
0752     open(VOMS, "voms-proxy-info |");
0753     $VOMS_SUCCESS = 0;
0754     while (<VOMS>){
0755     if (/timeleft/){$VOMS_SUCCESS=1;}
0756     }
0757
0758     if ($VOMS_SUCCESS){
0759     print "INFO:  check of voms-proxy=info succeeded.\n";
0760     }
0761
0762     if (! $VOMS_SUCCESS) {
0763     print "ERROR: you must call voms-proxy-init first, in order to access remote files for tests!\n";
0764     return;
0765     }
0766
0767
0768     if ($FAST){
0769     print "INFO: fast mode was specified... note that successful results will not green-light a commit.\n";
0770     }
0771
0772     if (! ($RECYCLE || $REDO)){
0773     if (-e $WORK_DIR){
0774         if (!$DELETE){
0775         print "ERROR: cowardly refusing to overwrite existing test directory: $WORK_DIR\n";
0776         print "ERROR: (move or delete it yourself)\n";
0777         return;
0778         } else {
0779         system "rm -fr $WORK_DIR";
0780         if (-e $WORK_DIR){
0781             print "ERROR: could not delete $WORK_DIR\n";
0782             return;
0783         }
0784         }
0785     }
0786     system "mkdir $WORK_DIR";
0787     if (! -e $WORK_DIR){
0788         print "ERROR: could not create $WORK_DIR\n";
0789         return;
0790     }
0791     } else {
0792     if (! -e $WORK_DIR){
0793         print "ERROR: --recycle or --redo specified but $WORK_DIR does not exist yet....\n";
0794         return;
0795     }
0796     }
0797
0798
0799     $start_time = time();
0800
0801     chdir $WORK_DIR;
0802     $PWD = `pwd`;
0803     chomp $PWD;
0804     $LOG = "$PWD/$MAIN_LOG";
0805
0806     # fork off and die:
0807     my $pid = fork();
0808     die "$0: fork: $!" unless defined $pid;
0809     if ($pid) {
0810     print "INFO: Coffee Time!!!\n";
0811     if ($SLOW){ print "INFO:  Slow mode specified... better make it two!\n"; }
0812     print "INFO: You can view progress by running:\n";
0813     print "tail -f $LOG\n";
0814     exit(0);
0815     }
0816     open STDOUT,">",$LOG or die $!;
0817     open STDERR,">",$LOG or die $!;
0818
0819     print "INFO: launching $NUM_JOBS test jobs...\n" if (! $SINGLE );
0820     $PIDS   = ();
0821     $IJOB   = ();
0822     for ($ijob = 0; $ijob < $NUM_JOBS; $ijob++){
0823     next if ($SINGLE && ($ijob != $SINGLE_JOB));
0824     if ($REDO) {
0825         $SUCCESSFILE = "test_$ijob/SUCCESS";
0826         if (-e $SUCCESSFILE) { print "INFO:  tests job $ijob was already successful... skipping\n"; next; }
0827         else { if (-e "test_$ijob") { system "rm -fr test_$ijob"; } }
0828     }
0829     print "INFO: launching job number $ijob\n";
0830     my $pid = fork();
0831     die "$0: fork: $!" unless defined $pid;
0832
0833     if ($pid) {
0834         print "INFO: child process $pid launched...\n";
0835         push @PIDS, $pid;
0836         push @IJOB, $ijob;
0837     } else {
0838         run_job($ijob);
0839     }
0840     }
0841
0842     sub kill_children {
0843     foreach $pid (@PIDS) {
0844         print "ERROR: killing child process $pid\n";
0845         kill HUP => $pid;
0846     }
0847     print "ERROR: exiting before all jobs complete..\n";
0848     };
0849
0850     $SIG{ALRM} = sub {
0851     print "ERROR: timeout waiting for child processes...\n";
0852     kill_children();
0853     summary();
0854     exit 0;
0855     };
0856     alarm $TIMEOUT;
0857
0858     while($#PIDS >= 0){
0859     if (-e $DIE_FILE){
0860         print "INFO: received DIE command... killing child processes and exiting...\n";
0861         system "rm $DIE_FILE";
0862         kill_children();
0863         summary();
0864         exit 0;
0865     }
0866
0867     for ($i=0; $i <= $#PIDS; $i++){
0868         #print "DEBUG: $i\n";
0869         $pid = waitpid ($PIDS[$i], &POSIX::WNOHANG);
0870         #print "DEBUG: pid $pid\n";
0871         if ($pid > 0){
0872         $SUCCESSFILE = "test_$IJOB[$i]/SUCCESS";
0873         if (-e $SUCCESSFILE) { print "INFO:  job $IJOB[$i] (pid $pid) has exited with status:  SUCCESS\n"; }
0874         else { print "INFO:  job $i (pid $pid) has exited with status:  FAILURE\n"; }
0875         splice(@PIDS, $i, 1);
0876         splice(@IJOB, $i, 1);
0877         $i--;
0878         }
0879     }
0880     system "sleep 1"
0881     }
0882     alarm 0;
0883     print "INFO: all jobs have finished.\n";
0884
0885     $job_time = time() - $start_time;
0886     print "INFO: testL1T took $job_time seconds to complete.\n";
0887     summary();
0888
0889
0890
0891 }
0892
0893 sub summary {
0894     $FAIL = 0;
0895     for ($ijob = 0; $ijob < $NUM_JOBS; $ijob++){
0896     next if ($SINGLE && ($ijob != $SINGLE_JOB));
0897     $SUCCESSFILE = "test_$ijob/SUCCESS";
0898     if (-e $SUCCESSFILE) { print "STATUS:  tests job $ijob result was SUCCESS\n"; }
0899     else                 {
0900         print "STATUS:  tests job $ijob result was FAIL\n";
0901         $FAIL = 1;
0902     }
0903     }
0904     if ($FAIL) { print "STATUS:  testL1T overall status:  FAIL\n"; }
0905     else       {
0906     print "STATUS:  testL1T overall status:  SUCCESS\n";
0907     if ($FAST || $DRYRUN || $RECYCLE || $REDO || $SINGLE){
0908         print "STATUS:  results not sufficient for greenlight to commit due to --fast, --dryrun, --redo, --single, or --recycle option\n";
0909     }
0910     else {
0911         print "STATUS:  you have GREEN LIGHT to commit your L1T code!\n";
0912         system "touch GREEN_LIGHT";
0913     }
0914     }
0915 }