Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2021-02-14 12:47:37

0001 #
0002 #
0003 #   Calibration Update Script
0004 #
0005 # Authors: Tom Nummy     Northeastern University
0006 #          Oana Boeriu   Northeastern University 
0007 #
0008 # Description: This script's purpose is to work towards automation of the
0009 #              generation, validation, and updating of calbration constants
0010 #              for CSC gains, pedestals, crosstalk, and the noise matrix
0011 #
0012 #
0013 #============================================================================
0014 
0015 #### MAKE SURE YOU HAVE THE MOST RECENT DB VALUES FIRST!!! ####
0016 
0017 #### Run readDBGains_cfg.py, readDBPedestals_cfg.py, readDBNoiseMatrix_cfg.py,
0018 #### and readDBCrosstalk_cfg.py to be sure.
0019 
0020 echo ==================================================================
0021 echo "Beginning new Calibration Update"
0022 echo ==================================================================
0023 echo
0024 sleep 3
0025 echo "Checking the CFEB_04 run log"
0026 echo 
0027 sleep 2
0028 
0029 #--- Update the directory list ----
0030 ls -l -t -d /data/dqm/calib/Test_CFEB04/run*.plots | awk '{print $9}' > /nfshome0/nummy/Logs/CFEB_04/Test_CFEB04_dirlist.dat
0031 
0032 #--- Compile and run the Log checker to find if there are any unprocessed runs in the directory ---
0033 g++ /nfshome0/nummy/Logs/CFEB_04/LogChecker_CFEB04.cpp -o /nfshome0/nummy/Logs/CFEB_04/LC04
0034 /nfshome0/nummy/Logs/CFEB_04/LC04
0035 
0036 #--- Compile the C++ validator for later (outside loop b/c we only need to compile once) ---
0037 g++ readGains.cpp -o readGains
0038 chmod +x readGains
0039 
0040 #echo
0041 #echo "here's the job list:"
0042 #cat /nfshome0/nummy/Logs/CFEB_04/tmpruns_04.dat
0043 
0044 #--- This is the list of folders in the directory that haven't been processed yet (the job list) ---
0045 #runqueue="/nfshome0/nummy/Logs/CFEB_04/Test_CFEB04_dirlist.dat"
0046 runqueue="/nfshome0/nummy/Logs/CFEB_04/tmpruns_04.dat"
0047 
0048 exec <$runqueue
0049 i=0
0050 #--- Looping over the job list --- 
0051 while read CFEB4RUNDIR
0052   do
0053   echo
0054   echo ----------------------------------------------------------
0055   echo
0056   echo "Processing " $CFEB4RUNDIR
0057   i=`expr $i + 1`
0058   echo
0059   #--- get all the ascii files from all the chambers and put into a list for the upcoming loop ---
0060   CFEB4ASCIIFILES=`ls -l $CFEB4RUNDIR/*/*_DB.dat | awk '{print $9}'`
0061   echo
0062   
0063   if [ "$CFEB4ASCIIFILES" != "" ]
0064       then
0065       #--- merging the ascii files into one summary file ---
0066       for asciifile in $CFEB4ASCIIFILES
0067         do
0068         cat $asciifile >> /nfshome0/nummy/merged_data/gainSummary.dat
0069       done
0070       echo "merged ASCII files"
0071       echo
0072   fi
0073   
0074   #--- Run the C++ validator ---
0075   sleep 2
0076   echo "Running gains validator, Please wait..."
0077   ./readGains
0078   sleep 2
0079   echo "...complete"
0080   echo
0081 
0082   #--- Until CMSSW is installed on the CSC-DQM machine we have to run all CMSSW programs on cmsusr0 ---  #ssh cmsusr0 ./getGainsDB.sh
0083   #mv dbgains.dat DBValues/
0084   
0085   #--- Run the CMSSW comparison analyzer ---
0086   cd /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test
0087   source /nfshome0/cmssw2/scripts/setup.sh
0088   eval `scramv1 runt -sh`
0089 
0090   echo "beginning CMSSW analyser, this may take a few moments"
0091   cmsRun gains_compare_cfg.py
0092   echo
0093   echo "Comparison complete!"
0094   echo
0095   # files need to be moved to correct path
0096   cp /nfshome0/nummy/Good_data/goodGains.dat /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test/gains.dat
0097   cp /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test/DBValues/dbgains.dat /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test/old_dbgains.dat
0098 
0099   #--- Generate SQlite Files ---
0100   #cmsRun CSCDBGainsPopCon_cfg.py
0101   cd ~
0102   
0103   
0104   #---Some housekeeping, making folders by run number and moving summary and C++ ouput data there ---
0105   newfoldername=`echo $CFEB4RUNDIR | awk '{ n=split($1,path,"/"); print path[n] }'`
0106   if [ -d /nfshome0/nummy/Good_data/$newfoldername ]
0107       then
0108       echo "The directory for the processed data already exists, overwriting previous entries"
0109   else
0110       mkdir /nfshome0/nummy/Good_data/$newfoldername
0111   fi 
0112   
0113   if [ -d /nfshome0/nummy/merged_data/$newfoldername ]
0114       then
0115       echo "The directory for the merged data already exists, overwriting previous entries"
0116   else
0117       mkdir /nfshome0/nummy/merged_data/$newfoldername
0118   fi 
0119   
0120   if [ -d /nfshome0/nummy/Diff_Output/$newfoldername ]
0121       then
0122       echo "The directory for the diff output already exists, overwriting previous entries"
0123       echo
0124   else
0125       mkdir /nfshome0/nummy/Diff_Output/$newfoldername
0126   fi
0127   
0128   #--- write buffers to the corresponding run folders
0129   cp /nfshome0/nummy/merged_data/gainSummary.dat /nfshome0/nummy/merged_data/$newfoldername/
0130   cp /nfshome0/nummy/Good_data/goodGains.dat /nfshome0/nummy/Good_data/$newfoldername/
0131   cp /nfshome0/nummy/Diff_Output/diffGains.dat /nfshome0/nummy/Diff_Output/$newfoldername/
0132 
0133   #--- clear the buffer for the next run ---
0134   echo "" > /nfshome0/nummy/merged_data/gainSummary.dat
0135   
0136 done
0137 
0138 echo "Processed " $i " gain calibration runs"
0139 sleep 3
0140 
0141 echo
0142 echo ==================================================
0143 echo "Getting ready to process Crosstalk runs"
0144 echo ==================================================
0145 echo
0146 sleep 2
0147 echo "Checking the CFEB_03 run log"
0148 echo
0149 sleep 2
0150 
0151 #--- Update the directory list ----
0152 ls -l -t -d /data/dqm/calib/Test_CFEB03/run*.plots | awk '{print $9}' > /nfshome0/nummy/Logs/CFEB_03/Test_CFEB03_dirlist.dat
0153 
0154 #--- Compile and run the Log checker to find if there are any unprocessed runs in the directory ---
0155 g++ /nfshome0/nummy/Logs/CFEB_03/LogChecker_CFEB03.cpp -o /nfshome0/nummy/Logs/CFEB_03/LC03
0156 /nfshome0/nummy/Logs/CFEB_03/LC03
0157 
0158 #--- Compile the C++ validator for later (outside loop b/c we only need to compile once) ---
0159 g++ readXtalk.cpp -o readXtalk
0160 chmod +x readXtalk
0161 
0162 #echo
0163 #echo "here's the job list:"
0164 #cat /nfshome0/nummy/Logs/CFEB_03/tmpruns_03.dat
0165 
0166 #--- This is the list of folders in the directory that haven't been processed yet (the job list) ---
0167 #runqueue="/nfshome0/nummy/Logs/CFEB_03/Test_CFEB03_dirlist.dat"
0168 runqueue="/nfshome0/nummy/Logs/CFEB_03/tmpruns_03.dat"
0169 
0170 exec <$runqueue
0171 i=0
0172 #--- Looping over the job list --- 
0173 while read CFEB3RUNDIR
0174   do
0175   echo
0176   echo ----------------------------------------------------------
0177   echo
0178   echo "Processing " $CFEB3RUNDIR
0179   i=`expr $i + 1`
0180   echo
0181   #--- get all the ascii files from all the chambers and put into a list for the upcoming loop ---
0182   CFEB3ASCIIFILES=`ls -l $CFEB3RUNDIR/*/*Xtalk.dat | awk '{print $9}'`
0183   echo
0184   
0185   if [ "$CFEB3ASCIIFILES" != "" ]
0186       then
0187       echo "Found the crosstalk ascii files"
0188       #--- merging the ascii files into one summary file ---
0189       for asciifile in $CFEB3ASCIIFILES
0190         do
0191         cat $asciifile >> /nfshome0/nummy/merged_data/xtalkSummary.dat
0192       done
0193   fi
0194   echo "merged ASCII files"
0195   echo
0196   
0197   #--- Run the C++ validator ---
0198   sleep 2
0199   echo "Running XTalk validator, Please wait..."
0200   ./readXtalk
0201   sleep 2
0202   echo "...complete"
0203   echo
0204 
0205   #--- Run the CMSSW comparison analyzer ---
0206   cd /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test
0207   source /nfshome0/cmssw2/scripts/setup.sh
0208   eval `scramv1 runt -sh`
0209 
0210   echo "beginning CMSSW analyser, this may take a few moments"
0211   cmsRun xtalk_compare_cfg.py
0212   echo
0213   echo "Comparison complete!"
0214   echo
0215   # move files to correct paths
0216   cp /nfshome0/nummy/Good_data/goodXtalk.dat /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test/xtalk.dat
0217   cp /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test/DBValues/dbxtalk.dat /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test/old_dbxtalk.dat
0218 
0219   #--- Generate SQlite Files ---
0220   #cmsRun CSCDBCrosstalkPopCon_cfg.py
0221   cd ~
0222   
0223   #---Some housekeeping, making folders by run number and moving summary and C++ ouput data there ---
0224   newfoldername=`echo $CFEB3RUNDIR | awk '{ n=split($1,path,"/"); print path[n] }'`
0225   if [ -d /nfshome0/nummy/Good_data/$newfoldername ]
0226       then
0227       echo "The directory for the processed data already exists, overwriting previous entries"
0228   else
0229       mkdir /nfshome0/nummy/Good_data/$newfoldername
0230   fi 
0231   
0232   if [ -d /nfshome0/nummy/merged_data/$newfoldername ]
0233       then
0234       echo "The directory for the merged data already exists, overwriting previous entries"
0235   else
0236       mkdir /nfshome0/nummy/merged_data/$newfoldername
0237   fi 
0238   
0239   if [ -d /nfshome0/nummy/Diff_Output/$newfoldername ]
0240       then
0241       echo "The directory for the diff output data already exists, overwriting previous entries"
0242       echo
0243   else
0244       mkdir /nfshome0/nummy/Diff_Output/$newfoldername
0245   fi
0246   
0247   cp /nfshome0/nummy/merged_data/xtalkSummary.dat /nfshome0/nummy/merged_data/$newfoldername/
0248   cp /nfshome0/nummy/Good_data/goodXtalk.dat /nfshome0/nummy/Good_data/$newfoldername/
0249   cp /nfshome0/nummy/Diff_Output/diffXtalk.dat /nfshome0/nummy/Diff_Output/$newfoldername/
0250   
0251   echo "" > /nfshome0/nummy/merged_data/xtalkSummary.dat
0252   
0253 done
0254 
0255 echo "Processed " $i " crosstalk calibration runs"
0256 sleep 2
0257 
0258 echo
0259 echo ==================================================
0260 echo "Getting ready to process Pedestal/Noise Matrix runs"
0261 echo ==================================================
0262 echo
0263 sleep 3
0264 echo "Checking the CFEB_02 log"
0265 echo
0266 sleep 2
0267 
0268 #--- Update the directory list ----
0269 ls -l -t -d /data/dqm/calib/Test_CFEB02/run*.plots | awk '{print $9}' > /nfshome0/nummy/Logs/CFEB_02/Test_CFEB02_dirlist.dat
0270 
0271 #--- Compile and run the Log checker to find if there are any unprocessed runs in the directory ---
0272 g++ /nfshome0/nummy/Logs/CFEB_02/LogChecker_CFEB02.cpp -o /nfshome0/nummy/Logs/CFEB_02/LC02
0273 /nfshome0/nummy/Logs/CFEB_02/LC02
0274 
0275 #--- Compile the C++ validator for later (outside loop b/c we only need to compile once) ---
0276 g++ readPeds.cpp -o readPeds
0277 chmod +x readPeds
0278 
0279 g++ readMatrix.cpp -o readMatrix
0280 chmod +x readMatrix
0281 
0282 #echo
0283 #echo "here's the job list:"
0284 #cat /nfshome0/nummy/Logs/CFEB_02/tmpruns_02.dat
0285 
0286 #--- This is the list of folders in the directory that haven't been processed yet (the job list) ---
0287 #runqueue="/nfshome0/nummy/Logs/CFEB_02/Test_CFEB02_dirlist.dat"
0288 runqueue="/nfshome0/nummy/Logs/CFEB_02/tmpruns_02.dat"
0289 
0290 exec <$runqueue
0291 i=0
0292 #--- Looping over the job list --- 
0293 while read CFEB2RUNDIR
0294   do
0295   echo
0296   echo ----------------------------------------------------------
0297   echo
0298   echo "Processing " $CFEB2RUNDIR
0299   i=`expr $i + 1`
0300   echo
0301   #--- get all the ascii files from all the chambers and put into a list for the upcoming loop ---
0302   CFEB2ASCIIFILES=`ls -l $CFEB2RUNDIR/*/*_DB.dat | awk '{print $9}'`
0303   echo
0304   
0305   if [ "$CFEB2ASCIIFILES" != "" ]
0306       then
0307       #--- merging the ascii files into one summary file ---
0308       for asciifile in $CFEB2ASCIIFILES
0309         do
0310         cat $asciifile >> /nfshome0/nummy/merged_data/pedSummary.dat
0311       done
0312   fi
0313   echo "merged pedestal ASCII files"
0314   echo
0315   
0316   #--- Run the C++ validator ---
0317   sleep 2
0318   echo "Running Pedestal validator, Please wait..."
0319   ./readPeds
0320   sleep 2
0321   echo "...complete"
0322   echo
0323   
0324   echo "goodPeds.dat:"
0325   head /nfshome0/nummy/Good_data/goodPeds.dat
0326   echo
0327   echo "dbpeds.dat:"
0328   head /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test/DBValues/dbpeds.dat
0329   echo
0330 
0331    #--- Run the CMSSW comparison analyzer ---
0332   cd /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test
0333   source /nfshome0/cmssw2/scripts/setup.sh
0334   eval `scramv1 runt -sh`
0335 
0336   echo "beginning CMSSW analyser, this may take a few moments"
0337   cmsRun peds_compare_cfg.py
0338   echo
0339   echo "Comparison complete!"
0340   echo
0341    # files need to be moved to correct path
0342   cp /nfshome0/nummy/Good_data/goodPeds.dat /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test/peds.dat
0343   cp /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test/DBValues/dbpeds.dat /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test/old_dbpeds.dat
0344 
0345   #--- Generate SQlite Files ---
0346   #cmsRun CSCDBPedestalsPopCon_cfg.py
0347   cd ~
0348   
0349   #---Some housekeeping, making folders by run number and moving summary and C++ ouput data there ---
0350   newfoldername=`echo $CFEB2RUNDIR | awk '{ n=split($1,path,"/"); print path[n] }'`
0351   if [ -d /nfshome0/nummy/Good_data/$newfoldername ]
0352       then
0353       echo "The directory for the processed data already exists, overwriting previous entries"
0354   else
0355       mkdir /nfshome0/nummy/Good_data/$newfoldername
0356   fi 
0357   
0358   if [ -d /nfshome0/nummy/merged_data/$newfoldername ]
0359       then
0360       echo "The directory for the merged data already exists, overwriting previous entries"
0361   else
0362       mkdir /nfshome0/nummy/merged_data/$newfoldername
0363   fi 
0364   
0365   if [ -d /nfshome0/nummy/Diff_Output/$newfoldername ]
0366       then
0367       echo "The directory for the diff output  already exists, overwriting previous entries"
0368       echo
0369   else
0370       mkdir /nfshome0/nummy/Diff_Output/$newfoldername
0371   fi
0372 
0373   cp /nfshome0/nummy/merged_data/pedSummary.dat /nfshome0/nummy/merged_data/$newfoldername/
0374   cp /nfshome0/nummy/Good_data/goodPeds.dat /nfshome0/nummy/Good_data/$newfoldername/
0375   cp /nfshome0/nummy/Diff_Output/diffPeds.dat /nfshome0/nummy/Diff_Output/$newfoldername/
0376 
0377   echo "" > /nfshome0/nummy/merged_data/pedSummary.dat
0378   
0379 done
0380 
0381 echo "Processed " $i " Pedestal calibration runs"
0382 sleep 2
0383 
0384 #----------------------------------------------------------------
0385 
0386 exec <$runqueue
0387 i=0
0388 #--- Looping over the job list --- 
0389 while read CFEB2RUNDIR
0390   do
0391   echo
0392   echo ----------------------------------------------------------
0393   echo
0394   echo "Processing " $CFEB2RUNDIR
0395   i=`expr $i + 1`
0396   echo
0397   #--- get all the ascii files from all the chambers and put into a list for the upcoming loop ---
0398   CFEB2ASCIIFILES=`ls -l $CFEB2RUNDIR/*/*_DB_NoiseMatrix.dat | awk '{print $9}'`
0399   echo
0400   
0401   if [ "$CFEB2ASCIIFILES" != "" ]
0402       then
0403       #--- merging the ascii files into one summary file ---
0404       for asciifile in $CFEB2ASCIIFILES
0405         do
0406         cat $asciifile >> /nfshome0/nummy/merged_data/matrixSummary.dat
0407       done
0408   fi
0409   echo "merged noise matrix ASCII files"
0410   echo
0411   
0412   #--- Run the C++ validator ---
0413   sleep 2
0414   echo "Running Noise Matrix validator, Please wait..."
0415   ./readMatrix
0416   sleep 2
0417   echo "...complete"
0418   echo
0419   
0420   echo "goodMatrix.dat:"
0421   head /nfshome0/nummy/Good_data/goodMatrix.dat
0422   echo
0423   echo "dbmatrix.dat:"
0424   head /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test/DBValues/dbmatrix.dat
0425   echo
0426   
0427   #--- Run the CMSSW comparison analyzer ---
0428   cd /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test
0429   source /nfshome0/cmssw2/scripts/setup.sh
0430   eval `scramv1 runt -sh`
0431 
0432   echo "beginning CMSSW analyser, this may take a few moments"
0433   cmsRun noisematrix_compare_cfg.py
0434   echo
0435   echo "Comparison complete!"
0436   echo
0437    # files need to be moved to correct path
0438   cp /nfshome0/nummy/Good_data/goodMatrix.dat /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test/matrix.dat
0439   cp /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test/DBValues/dbmatrix.dat /nfshome0/nummy/CMSSW_3_2_5/src/CalibMuon/CSCCalibration/test/old_dbmatrix.dat
0440 
0441   #--- Generate SQlite Files ---
0442   #cmsRun CSCDBNoiseMatrixPopCon_cfg.py
0443   cd ~
0444 
0445 
0446   #---Some housekeeping, making folders by run number and moving summary, C++, and Diff ouput data there ---
0447   newfoldername=`echo $CFEB2RUNDIR | awk '{ n=split($1,path,"/"); print path[n] }'`
0448   if [ -d /nfshome0/nummy/Good_data/$newfoldername ]
0449       then
0450       echo "The directory for the processed data already exists, overwriting previous entries"
0451   else
0452       mkdir /nfshome0/nummy/Good_data/$newfoldername
0453   fi 
0454   
0455   if [ -d /nfshome0/nummy/merged_data/$newfoldername ]
0456       then
0457       echo "The directory for the merged data already exists, overwriting previous entries"
0458   else
0459       mkdir /nfshome0/nummy/merged_data/$newfoldername
0460   fi 
0461 
0462   if [ -d /nfshome0/nummy/Diff_Output/$newfoldername ]
0463       then
0464       echo "The directory for the diff output  already exists, overwriting previous entries"
0465       echo
0466   else
0467       mkdir /nfshome0/nummy/Diff_Output/$newfoldername
0468   fi
0469   
0470   cp /nfshome0/nummy/merged_data/matrixSummary.dat /nfshome0/nummy/merged_data/$newfoldername/
0471   cp /nfshome0/nummy/Good_data/goodMatrix.dat /nfshome0/nummy/Good_data/$newfoldername/
0472   cp /nfshome0/nummy/Diff_Output/diffMatrix.dat /nfshome0/nummy/Diff_Output/$newfoldername/
0473 
0474   echo "" > /nfshome0/nummy/merged_data/matrixSummary.dat
0475   
0476 done
0477 
0478 echo "Processed " $i " noise matrix calibration runs"
0479 sleep 2
0480 
0481 echo
0482 echo "Finished Calibration update.."
0483 echo
0484 sleep 2
0485 #cat einstein.dat