Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2022-03-03 02:23:12

0001 #! /usr/bin/env python3
0002 
0003 from __future__ import print_function
0004 from builtins import range
0005 import os, sys, optparse, math
0006 
0007 copyargs = sys.argv[:]
0008 for i in range(len(copyargs)):
0009     if copyargs[i] == "":
0010         copyargs[i] = "\"\""
0011     if copyargs[i].find(" ") != -1:
0012         copyargs[i] = "\"%s\"" % copyargs[i]
0013 commandline = " ".join(copyargs)
0014 
0015 prog = sys.argv[0]
0016 
0017 usage = """./%(prog)s DIRNAME ITERATIONS INITIALGEOM INPUTFILES [options]
0018 
0019 Creates (overwrites) a directory for each of the iterations and creates (overwrites)
0020 submitJobs.sh with the submission sequence and dependencies.
0021 
0022 DIRNAME        directories will be named DIRNAME01, DIRNAME02, etc.
0023 ITERATIONS     number of iterations
0024 INITIALGEOM    SQLite file containing muon geometry with tag names
0025                DTAlignmentRcd, DTAlignmentErrorExtendedRcd, CSCAlignmentRcd, CSCAlignmentErrorExtendedRcd
0026 INPUTFILES     Python file defining 'fileNames', a list of input files as
0027                strings (create with findQualityFiles.py)""" % vars()
0028 
0029 parser = optparse.OptionParser(usage)
0030 parser.add_option("-j", "--jobs",
0031                    help="approximate number of \"gather\" subjobs",
0032                    type="int",
0033                    default=50,
0034                    dest="subjobs")
0035 parser.add_option("-s", "--submitJobs",
0036                    help="alternate name of submitJobs.sh script (please include .sh extension); a file with this name will be OVERWRITTEN",
0037                    type="string",
0038                    default="submitJobs.sh",
0039                    dest="submitJobs")
0040 parser.add_option("-b", "--big",
0041                   help="if invoked, subjobs will also be run on cmscaf1nd",
0042                   action="store_true",
0043                   dest="big")
0044 parser.add_option("-u", "--user_mail",
0045                   help="if invoked, send mail to a specified email destination. If \"-u\" is not present, the default destination LSB_MAILTO in lsf.conf will be used",
0046                   type="string",
0047                   dest="user_mail")
0048 parser.add_option("--mapplots",
0049                   help="if invoked, draw \"map plots\"",
0050                   action="store_true",
0051                   dest="mapplots")
0052 parser.add_option("--segdiffplots",
0053                   help="if invoked, draw \"segment-difference plots\"",
0054                   action="store_true",
0055                   dest="segdiffplots")
0056 parser.add_option("--curvatureplots",
0057                   help="if invoked, draw \"curvature plots\"",
0058                   action="store_true",
0059                   dest="curvatureplots")
0060 parser.add_option("--globalTag",
0061                   help="GlobalTag for alignment/calibration conditions (typically all conditions except muon and tracker alignment)",
0062                   type="string",
0063                   default="CRAFT0831X_V1::All",
0064                   dest="globaltag")
0065 parser.add_option("--trackerconnect",
0066                   help="connect string for tracker alignment (frontier://FrontierProd/CMS_COND_310X_ALIGN or sqlite_file:...)",
0067                   type="string",
0068                   default="",
0069                   dest="trackerconnect")
0070 parser.add_option("--trackeralignment",
0071                   help="name of TrackerAlignmentRcd tag",
0072                   type="string",
0073                   default="Alignments",
0074                   dest="trackeralignment")
0075 parser.add_option("--trackerAPEconnect",
0076                   help="connect string for tracker APEs (frontier://... or sqlite_file:...)",
0077                   type="string",
0078                   default="",
0079                   dest="trackerAPEconnect")
0080 parser.add_option("--trackerAPE",
0081                   help="name of TrackerAlignmentErrorExtendedRcd tag (tracker APEs)",
0082                   type="string",
0083                   default="AlignmentErrorsExtended",
0084                   dest="trackerAPE")
0085 parser.add_option("--trackerBowsconnect",
0086                   help="connect string for tracker Surface Deformations (frontier://... or sqlite_file:...)",
0087                   type="string",
0088                   default="",
0089                   dest="trackerBowsconnect")
0090 parser.add_option("--trackerBows",
0091                   help="name of TrackerSurfaceDeformationRcd tag",
0092                   type="string",
0093                   default="TrackerSurfaceDeformations",
0094                   dest="trackerBows")
0095 parser.add_option("--gprcdconnect",
0096                   help="connect string for GlobalPositionRcd (frontier://... or sqlite_file:...)",
0097                   type="string",
0098                   default="",
0099                   dest="gprcdconnect")
0100 parser.add_option("--gprcd",
0101                   help="name of GlobalPositionRcd tag",
0102                   type="string",
0103                   default="GlobalPosition",
0104                   dest="gprcd")
0105 parser.add_option("--iscosmics",
0106                   help="if invoked, use cosmic track refitter instead of the standard one",
0107                   action="store_true",
0108                   dest="iscosmics")
0109 parser.add_option("--station123params",
0110                   help="alignable parameters for DT stations 1, 2, 3 (see SWGuideAlignmentAlgorithms#Selection_of_what_to_align)",
0111                   type="string",
0112                   default="111111",
0113                   dest="station123params")
0114 parser.add_option("--station4params",
0115                   help="alignable parameters for DT station 4",
0116                   type="string",
0117                   default="100011",
0118                   dest="station4params")
0119 parser.add_option("--cscparams",
0120                   help="alignable parameters for CSC chambers",
0121                   type="string",
0122                   default="100011",
0123                   dest="cscparams")
0124 parser.add_option("--minTrackPt",
0125                   help="minimum allowed track transverse momentum (in GeV)",
0126                   type="string",
0127                   default="0",
0128                   dest="minTrackPt")
0129 parser.add_option("--maxTrackPt",
0130                   help="maximum allowed track transverse momentum (in GeV)",
0131                   type="string",
0132                   default="1000",
0133                   dest="maxTrackPt")
0134 parser.add_option("--minTrackP",
0135                   help="minimum allowed track momentum (in GeV)",
0136                   type="string",
0137                   default="0",
0138                   dest="minTrackP")
0139 parser.add_option("--maxTrackP",
0140                   help="maximum allowed track momentum (in GeV)",
0141                   type="string",
0142                   default="10000",
0143                   dest="maxTrackP")
0144 parser.add_option("--minTrackerHits",
0145                   help="minimum number of tracker hits",
0146                   type="int",
0147                   default=15,
0148                   dest="minTrackerHits")
0149 parser.add_option("--maxTrackerRedChi2",
0150                   help="maximum tracker chi^2 per degrees of freedom",
0151                   type="string",
0152                   default="10",
0153                   dest="maxTrackerRedChi2")
0154 parser.add_option("--notAllowTIDTEC",
0155                   help="if invoked, do not allow tracks that pass through the tracker's TID||TEC region (not recommended)",
0156                   action="store_true",
0157                   dest="notAllowTIDTEC")
0158 parser.add_option("--twoBin",
0159                   help="if invoked, apply the \"two-bin method\" to control charge-antisymmetric errors",
0160                   action="store_true",
0161                   dest="twoBin")
0162 parser.add_option("--weightAlignment",
0163                   help="if invoked, segments will be weighted by ndf/chi^2 in the alignment",
0164                   action="store_true",
0165                   dest="weightAlignment")
0166 parser.add_option("--minAlignmentSegments",
0167                   help="minimum number of segments required to align a chamber",
0168                   type="int",
0169                   default=5,
0170                   dest="minAlignmentHits")
0171 parser.add_option("--notCombineME11",
0172                   help="if invoced, treat ME1/1a and ME1/1b as separate objects",
0173                   action="store_true",
0174                   dest="notCombineME11")
0175 parser.add_option("--maxEvents",
0176                   help="maximum number of events",
0177                   type="string",
0178                   default="-1",
0179                   dest="maxEvents")
0180 parser.add_option("--skipEvents",
0181                   help="number of events to be skipped",
0182                   type="string",
0183                   default="0",
0184                   dest="skipEvents")
0185 parser.add_option("--validationLabel",
0186                   help="if given nonempty string RUNLABEL, diagnostics and creation of plots will be run in the end of the last iteration; the RUNLABEL will be used to mark a run; the results will be put into a RUNLABEL_DATESTAMP.tgz tarball",
0187                   type="string",
0188                   default="",
0189                   dest="validationLabel")
0190 parser.add_option("--maxResSlopeY",
0191                   help="maximum residual slope y component",
0192                   type="string",
0193                   default="10",
0194                   dest="maxResSlopeY")
0195 parser.add_option("--motionPolicyNSigma",
0196                   help="minimum nsigma(deltax) position displacement in order to move a chamber for the final alignment result; default NSIGMA=3",
0197                   type="int",
0198                   default=3,
0199                   dest="motionPolicyNSigma")
0200 parser.add_option("--noCleanUp",
0201                   help="if invoked, temporary plotting???.root and *.tmp files would not be removed at the end of each align job",
0202                   action="store_true",
0203                   dest="noCleanUp")
0204 parser.add_option("--noCSC",
0205                   help="if invoked, CSC endcap chambers would not be processed",
0206                   action="store_true",
0207                   dest="noCSC")
0208 parser.add_option("--noDT",
0209                   help="if invoked, DT barrel chambers would not be processed",
0210                   action="store_true",
0211                   dest="noDT")
0212 parser.add_option("--createMapNtuple",
0213                   help="if invoked while mapplots are switched on, a special ntuple would be created",
0214                   action="store_true",
0215                   dest="createMapNtuple")
0216 parser.add_option("--inputInBlocks",
0217                   help="if invoked, assume that INPUTFILES provides a list of files already groupped into job blocks, -j has no effect in that case",
0218                   action="store_true",
0219                   dest="inputInBlocks")
0220 parser.add_option("--json",
0221                   help="If present with JSON file as argument, use JSON file for good lumi mask. "+\
0222                   "The latest JSON file is available at /afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions11/7TeV/Prompt/",
0223                   type="string",
0224                   default="",
0225                   dest="json")
0226 parser.add_option("--createAlignNtuple",
0227                   help="if invoked, debug ntuples with residuals would be created during gather jobs",
0228                   action="store_true",
0229                   dest="createAlignNtuple")
0230 parser.add_option("--residualsModel",
0231                   help="functional residuals model. Possible vaslues: pureGaussian2D (default), pureGaussian, GaussPowerTails, ROOTVoigt, powerLawTails",
0232                   type="string",
0233                   default="pureGaussian2D",
0234                   dest="residualsModel")
0235 parser.add_option("--useResiduals",
0236                   help="select residuals to use, possible values: 1111, 1110, 1100, 1010, 0010 that correspond to x y dxdz dydz residuals",
0237                   type="string",
0238                   default="1110",
0239                   dest="useResiduals")
0240 parser.add_option("--peakNSigma",
0241                   help="if >0, only residuals peaks within n-sigma multidimentional ellipsoid would be considered in the alignment fit",
0242                   type="string",
0243                   default="-1.",
0244                   dest="peakNSigma")
0245 parser.add_option("--preFilter",
0246                   help="if invoked, MuonAlignmentPreFilter module would be invoked in the Path's beginning. Can significantly speed up gather jobs.",
0247                   action="store_true",
0248                   dest="preFilter")
0249 parser.add_option("--muonCollectionTag",
0250                   help="If empty, use trajectories. If not empty, it's InputTag of muons collection to use in tracker muons based approach, e.g., 'newmuons' or 'muons'",
0251                   type="string",
0252                   default="",
0253                   dest="muonCollectionTag")
0254 parser.add_option("--maxDxy",
0255                   help="maximum track impact parameter with relation to beamline",
0256                   type="string",
0257                   default="1000.",
0258                   dest="maxDxy")
0259 parser.add_option("--minNCrossedChambers",
0260                   help="minimum number of muon chambers that a track is required to cross",
0261                   type="string",
0262                   default="3",
0263                   dest="minNCrossedChambers")
0264 parser.add_option("--extraPlots",
0265                   help="produce additional plots with geometry, reports differences, and corrections visulizations",
0266                   action="store_true",
0267                   dest="extraPlots")
0268 
0269 if len(sys.argv) < 5:
0270     raise SystemError("Too few arguments.\n\n"+parser.format_help())
0271 
0272 DIRNAME = sys.argv[1]
0273 ITERATIONS = int(sys.argv[2])
0274 INITIALGEOM = sys.argv[3]
0275 INPUTFILES = sys.argv[4]
0276 
0277 options, args = parser.parse_args(sys.argv[5:])
0278 user_mail = options.user_mail
0279 mapplots_ingeneral = options.mapplots
0280 segdiffplots_ingeneral = options.segdiffplots
0281 curvatureplots_ingeneral = options.curvatureplots
0282 globaltag = options.globaltag
0283 trackerconnect = options.trackerconnect
0284 trackeralignment = options.trackeralignment
0285 trackerAPEconnect = options.trackerAPEconnect
0286 trackerAPE = options.trackerAPE
0287 trackerBowsconnect = options.trackerBowsconnect
0288 trackerBows = options.trackerBows
0289 gprcdconnect = options.gprcdconnect
0290 gprcd = options.gprcd
0291 iscosmics = str(options.iscosmics)
0292 station123params = options.station123params
0293 station4params = options.station4params
0294 cscparams = options.cscparams
0295 muonCollectionTag = options.muonCollectionTag
0296 minTrackPt = options.minTrackPt
0297 maxTrackPt = options.maxTrackPt
0298 minTrackP = options.minTrackP
0299 maxTrackP = options.maxTrackP
0300 maxDxy = options.maxDxy
0301 minTrackerHits = str(options.minTrackerHits)
0302 maxTrackerRedChi2 = options.maxTrackerRedChi2
0303 minNCrossedChambers = options.minNCrossedChambers
0304 allowTIDTEC = str(not options.notAllowTIDTEC)
0305 twoBin = str(options.twoBin)
0306 weightAlignment = str(options.weightAlignment)
0307 minAlignmentHits = str(options.minAlignmentHits)
0308 combineME11 = str(not options.notCombineME11)
0309 maxEvents = options.maxEvents
0310 skipEvents = options.skipEvents
0311 validationLabel = options.validationLabel
0312 maxResSlopeY = options.maxResSlopeY
0313 theNSigma = options.motionPolicyNSigma
0314 residualsModel = options.residualsModel
0315 peakNSigma = options.peakNSigma
0316 preFilter = not not options.preFilter
0317 extraPlots = options.extraPlots
0318 useResiduals = options.useResiduals
0319 
0320 
0321 #print "check: ", allowTIDTEC, combineME11, preFilter
0322 
0323 doCleanUp = not options.noCleanUp
0324 createMapNtuple = not not options.createMapNtuple
0325 createAlignNtuple = not not options.createAlignNtuple
0326 
0327 doCSC = True
0328 if options.noCSC: doCSC = False
0329 doDT = True
0330 if options.noDT: doDT = False
0331 if options.noCSC and options.noDT:
0332     print("cannot do --noCSC and --noDT at the same time!")
0333     sys.exit()
0334 
0335 json_file = options.json
0336 
0337 fileNames=[]
0338 fileNamesBlocks=[]
0339 execfile(INPUTFILES)
0340 njobs = options.subjobs
0341 if (options.inputInBlocks):
0342     njobs = len(fileNamesBlocks)
0343     if njobs==0:
0344         print("while --inputInBlocks is specified, the INPUTFILES has no blocks!")
0345         sys.exit()
0346 
0347 stepsize = int(math.ceil(1.*len(fileNames)/options.subjobs))
0348 
0349 pwd = str(os.getcwd())
0350 
0351 copytrackerdb = ""
0352 if trackerconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % trackerconnect[12:]
0353 if trackerAPEconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % trackerAPEconnect[12:]
0354 if trackerBowsconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % trackerBowsconnect[12:]
0355 if gprcdconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % gprcdconnect[12:]
0356 
0357 
0358 #####################################################################
0359 # step 0: convert initial geometry to xml
0360 INITIALXML = INITIALGEOM + '.xml'
0361 if INITIALGEOM[-3:]=='.db':
0362     INITIALXML = INITIALGEOM[:-3] + '.xml'
0363 print("Converting",INITIALGEOM,"to",INITIALXML," ...will be done in several seconds...")
0364 print("./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py  %s %s --gprcdconnect %s --gprcd %s" % (INITIALGEOM,INITIALXML,gprcdconnect,gprcd))
0365 exit_code = os.system("./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py  %s %s --gprcdconnect %s --gprcd %s" % (INITIALGEOM,INITIALXML,gprcdconnect,gprcd))
0366 if exit_code>0:
0367     print("problem: conversion exited with code:", exit_code)
0368     sys.exit()
0369 
0370 #####################################################################
0371 
0372 def writeGatherCfg(fname, my_vars):
0373     file(fname, "w").write("""#/bin/sh
0374 # %(commandline)s
0375 
0376 export ALIGNMENT_CAFDIR=`pwd`
0377 
0378 cd %(pwd)s
0379 eval `scramv1 run -sh`
0380 export ALIGNMENT_AFSDIR=`pwd`
0381 
0382 export ALIGNMENT_INPUTFILES='%(inputfiles)s'
0383 export ALIGNMENT_ITERATION=%(iteration)d
0384 export ALIGNMENT_JOBNUMBER=%(jobnumber)d
0385 export ALIGNMENT_MAPPLOTS=%(mapplots)s
0386 export ALIGNMENT_SEGDIFFPLOTS=%(segdiffplots)s
0387 export ALIGNMENT_CURVATUREPLOTS=%(curvatureplots)s
0388 export ALIGNMENT_GLOBALTAG=%(globaltag)s
0389 export ALIGNMENT_INPUTDB=%(inputdb)s
0390 export ALIGNMENT_TRACKERCONNECT=%(trackerconnect)s
0391 export ALIGNMENT_TRACKERALIGNMENT=%(trackeralignment)s
0392 export ALIGNMENT_TRACKERAPECONNECT=%(trackerAPEconnect)s
0393 export ALIGNMENT_TRACKERAPE=%(trackerAPE)s
0394 export ALIGNMENT_TRACKERBOWSCONNECT=%(trackerBowsconnect)s
0395 export ALIGNMENT_TRACKERBOWS=%(trackerBows)s
0396 export ALIGNMENT_GPRCDCONNECT=%(gprcdconnect)s
0397 export ALIGNMENT_GPRCD=%(gprcd)s
0398 export ALIGNMENT_ISCOSMICS=%(iscosmics)s
0399 export ALIGNMENT_STATION123PARAMS=%(station123params)s
0400 export ALIGNMENT_STATION4PARAMS=%(station4params)s
0401 export ALIGNMENT_CSCPARAMS=%(cscparams)s
0402 export ALIGNMENT_MUONCOLLECTIONTAG=%(muonCollectionTag)s
0403 export ALIGNMENT_MINTRACKPT=%(minTrackPt)s
0404 export ALIGNMENT_MAXTRACKPT=%(maxTrackPt)s
0405 export ALIGNMENT_MINTRACKP=%(minTrackP)s
0406 export ALIGNMENT_MAXTRACKP=%(maxTrackP)s
0407 export ALIGNMENT_MAXDXY=%(maxDxy)s
0408 export ALIGNMENT_MINTRACKERHITS=%(minTrackerHits)s
0409 export ALIGNMENT_MAXTRACKERREDCHI2=%(maxTrackerRedChi2)s
0410 export ALIGNMENT_MINNCROSSEDCHAMBERS=%(minNCrossedChambers)s
0411 export ALIGNMENT_ALLOWTIDTEC=%(allowTIDTEC)s
0412 export ALIGNMENT_TWOBIN=%(twoBin)s
0413 export ALIGNMENT_WEIGHTALIGNMENT=%(weightAlignment)s
0414 export ALIGNMENT_MINALIGNMENTHITS=%(minAlignmentHits)s
0415 export ALIGNMENT_COMBINEME11=%(combineME11)s
0416 export ALIGNMENT_MAXEVENTS=%(maxEvents)s
0417 export ALIGNMENT_SKIPEVENTS=%(skipEvents)s
0418 export ALIGNMENT_MAXRESSLOPEY=%(maxResSlopeY)s
0419 export ALIGNMENT_DO_DT=%(doDT)s
0420 export ALIGNMENT_DO_CSC=%(doCSC)s
0421 export ALIGNMENT_JSON=%(json_file)s
0422 export ALIGNMENT_CREATEMAPNTUPLE=%(createMapNtuple)s
0423 #export ALIGNMENT_CREATEALIGNNTUPLE=%(createAlignNtuple)s
0424 export ALIGNMENT_PREFILTER=%(preFilter)s
0425 
0426 
0427 if [ \"zzz$ALIGNMENT_JSON\" != \"zzz\" ]; then
0428   cp -f $ALIGNMENT_JSON $ALIGNMENT_CAFDIR/
0429 fi
0430 
0431 cp -f %(directory)sgather_cfg.py %(inputdbdir)s%(inputdb)s %(copytrackerdb)s $ALIGNMENT_CAFDIR/
0432 cd $ALIGNMENT_CAFDIR/
0433 ls -l
0434 cmsRun gather_cfg.py
0435 ls -l
0436 cp -f *.tmp %(copyplots)s $ALIGNMENT_AFSDIR/%(directory)s
0437 """ % my_vars)
0438 
0439 #####################################################################
0440 
0441 def writeAlignCfg(fname, my_vars):
0442     file("%salign.sh" % directory, "w").write("""#!/bin/sh
0443 # %(commandline)s
0444 
0445 export ALIGNMENT_CAFDIR=`pwd`
0446 
0447 cd %(pwd)s
0448 eval `scramv1 run -sh`
0449 export ALIGNMENT_AFSDIR=`pwd`
0450 export ALIGNMENT_INPUTDB=%(inputdb)s
0451 export ALIGNMENT_ITERATION=%(iteration)d
0452 export ALIGNMENT_GLOBALTAG=%(globaltag)s
0453 export ALIGNMENT_TRACKERCONNECT=%(trackerconnect)s
0454 export ALIGNMENT_TRACKERALIGNMENT=%(trackeralignment)s
0455 export ALIGNMENT_TRACKERAPECONNECT=%(trackerAPEconnect)s
0456 export ALIGNMENT_TRACKERAPE=%(trackerAPE)s
0457 export ALIGNMENT_TRACKERBOWSCONNECT=%(trackerBowsconnect)s
0458 export ALIGNMENT_TRACKERBOWS=%(trackerBows)s
0459 export ALIGNMENT_GPRCDCONNECT=%(gprcdconnect)s
0460 export ALIGNMENT_GPRCD=%(gprcd)s
0461 export ALIGNMENT_ISCOSMICS=%(iscosmics)s
0462 export ALIGNMENT_STATION123PARAMS=%(station123params)s
0463 export ALIGNMENT_STATION4PARAMS=%(station4params)s
0464 export ALIGNMENT_CSCPARAMS=%(cscparams)s
0465 export ALIGNMENT_MINTRACKPT=%(minTrackPt)s
0466 export ALIGNMENT_MAXTRACKPT=%(maxTrackPt)s
0467 export ALIGNMENT_MINTRACKP=%(minTrackP)s
0468 export ALIGNMENT_MAXTRACKP=%(maxTrackP)s
0469 export ALIGNMENT_MINTRACKERHITS=%(minTrackerHits)s
0470 export ALIGNMENT_MAXTRACKERREDCHI2=%(maxTrackerRedChi2)s
0471 export ALIGNMENT_ALLOWTIDTEC=%(allowTIDTEC)s
0472 export ALIGNMENT_TWOBIN=%(twoBin)s
0473 export ALIGNMENT_WEIGHTALIGNMENT=%(weightAlignment)s
0474 export ALIGNMENT_MINALIGNMENTHITS=%(minAlignmentHits)s
0475 export ALIGNMENT_COMBINEME11=%(combineME11)s
0476 export ALIGNMENT_MAXRESSLOPEY=%(maxResSlopeY)s
0477 export ALIGNMENT_CLEANUP=%(doCleanUp)s
0478 export ALIGNMENT_CREATEALIGNNTUPLE=%(createAlignNtuple)s
0479 export ALIGNMENT_RESIDUALSMODEL=%(residualsModel)s
0480 export ALIGNMENT_PEAKNSIGMA=%(peakNSigma)s
0481 export ALIGNMENT_USERESIDUALS=%(useResiduals)s
0482 
0483 cp -f %(directory)salign_cfg.py %(inputdbdir)s%(inputdb)s %(directory)s*.tmp  %(copytrackerdb)s $ALIGNMENT_CAFDIR/
0484 
0485 export ALIGNMENT_PLOTTINGTMP=`find %(directory)splotting0*.root -maxdepth 1 -size +0 -print 2> /dev/null`
0486 
0487 # if it's 1st or last iteration, combine _plotting.root files into one:
0488 if [ \"$ALIGNMENT_ITERATION\" != \"111\" ] || [ \"$ALIGNMENT_ITERATION\" == \"%(ITERATIONS)s\" ]; then
0489   #nfiles=$(ls %(directory)splotting0*.root 2> /dev/null | wc -l)
0490   if [ \"zzz$ALIGNMENT_PLOTTINGTMP\" != \"zzz\" ]; then
0491     hadd -f1 %(directory)s%(director)s_plotting.root %(directory)splotting0*.root
0492     #if [ $? == 0 ] && [ \"$ALIGNMENT_CLEANUP\" == \"True\" ]; then rm %(directory)splotting0*.root; fi
0493   fi
0494 fi
0495 
0496 if [ \"$ALIGNMENT_CLEANUP\" == \"True\" ] && [ \"zzz$ALIGNMENT_PLOTTINGTMP\" != \"zzz\" ]; then
0497   rm $ALIGNMENT_PLOTTINGTMP
0498 fi
0499 
0500 cd $ALIGNMENT_CAFDIR/
0501 export ALIGNMENT_ALIGNMENTTMP=`find alignment*.tmp -maxdepth 1 -size +1k -print 2> /dev/null`
0502 ls -l
0503 
0504 cmsRun align_cfg.py
0505 cp -f MuonAlignmentFromReference_report.py $ALIGNMENT_AFSDIR/%(directory)s%(director)s_report.py
0506 cp -f MuonAlignmentFromReference_outputdb.db $ALIGNMENT_AFSDIR/%(directory)s%(director)s.db
0507 cp -f MuonAlignmentFromReference_plotting.root $ALIGNMENT_AFSDIR/%(directory)s%(director)s.root
0508 
0509 cd $ALIGNMENT_AFSDIR
0510 ./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %(directory)s%(director)s.db %(directory)s%(director)s.xml --noLayers --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
0511 
0512 export ALIGNMENT_ALIGNMENTTMP=`find %(directory)salignment*.tmp -maxdepth 1 -size +1k -print 2> /dev/null`
0513 if [ \"$ALIGNMENT_CLEANUP\" == \"True\" ] && [ \"zzz$ALIGNMENT_ALIGNMENTTMP\" != \"zzz\" ]; then
0514   rm $ALIGNMENT_ALIGNMENTTMP
0515   echo " "
0516 fi
0517 
0518 # if it's not 1st or last iteration, do some clean up:
0519 if [ \"$ALIGNMENT_ITERATION\" != \"1\" ] && [ \"$ALIGNMENT_ITERATION\" != \"%(ITERATIONS)s\" ]; then
0520   if [ \"$ALIGNMENT_CLEANUP\" == \"True\" ] && [ -e %(directory)s%(director)s.root ]; then
0521     rm %(directory)s%(director)s.root
0522   fi
0523 fi
0524 
0525 # if it's last iteration, apply chamber motion policy
0526 if [ \"$ALIGNMENT_ITERATION\" == \"%(ITERATIONS)s\" ]; then
0527   # convert this iteration's geometry into detailed xml
0528   ./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %(directory)s%(director)s.db %(directory)s%(director)s_extra.xml --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
0529   # perform motion policy 
0530   ./Alignment/MuonAlignmentAlgorithms/scripts/motionPolicyChamber.py \
0531       %(INITIALXML)s  %(directory)s%(director)s_extra.xml \
0532       %(directory)s%(director)s_report.py \
0533       %(directory)s%(director)s_final.xml \
0534       --nsigma %(theNSigma)s
0535   # convert the resulting xml into the final sqlite geometry
0536   ./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %(directory)s%(director)s_final.xml %(directory)s%(director)s_final.db --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
0537 fi
0538 
0539 """ % my_vars)
0540 
0541 #####################################################################
0542 
0543 def writeValidationCfg(fname, my_vars):
0544     file(fname, "w").write("""#!/bin/sh
0545 # %(commandline)s
0546 
0547 export ALIGNMENT_CAFDIR=`pwd`
0548 mkdir files
0549 mkdir out
0550 
0551 cd %(pwd)s
0552 eval `scramv1 run -sh`
0553 ALIGNMENT_AFSDIR=`pwd`
0554 ALIGNMENT_ITERATION=%(iteration)d
0555 ALIGNMENT_MAPPLOTS=None
0556 ALIGNMENT_SEGDIFFPLOTS=None
0557 ALIGNMENT_CURVATUREPLOTS=None
0558 ALIGNMENT_EXTRAPLOTS=%(extraPlots)s
0559 export ALIGNMENT_GPRCDCONNECT=%(gprcdconnect)s
0560 export ALIGNMENT_GPRCD=%(gprcd)s
0561 export ALIGNMENT_DO_DT=%(doDT)s
0562 export ALIGNMENT_DO_CSC=%(doCSC)s
0563 
0564 
0565 # copy the scripts to CAFDIR
0566 cd Alignment/MuonAlignmentAlgorithms/scripts/
0567 cp -f plotscripts.py $ALIGNMENT_CAFDIR/
0568 cp -f mutypes.py $ALIGNMENT_CAFDIR/
0569 cp -f alignmentValidation.py $ALIGNMENT_CAFDIR/
0570 cp -f phiedges_fitfunctions.C $ALIGNMENT_CAFDIR/
0571 cp -f createTree.py $ALIGNMENT_CAFDIR/
0572 cp -f signConventions.py $ALIGNMENT_CAFDIR/
0573 cp -f convertSQLiteXML.py $ALIGNMENT_CAFDIR/
0574 cp -f wrapperExtraPlots.sh $ALIGNMENT_CAFDIR/
0575 cd -
0576 cp Alignment/MuonAlignmentAlgorithms/test/browser/tree* $ALIGNMENT_CAFDIR/out/
0577 
0578 # copy the results to CAFDIR
0579 cp -f %(directory1)s%(director1)s_report.py $ALIGNMENT_CAFDIR/files/
0580 cp -f %(directory)s%(director)s_report.py $ALIGNMENT_CAFDIR/files/
0581 cp -f %(directory1)s%(director1)s.root $ALIGNMENT_CAFDIR/files/
0582 cp -f %(directory)s%(director)s.root $ALIGNMENT_CAFDIR/files/
0583 if [ -e %(directory1)s%(director1)s_plotting.root ] && [ -e %(directory)s%(director)s_plotting.root ]; then
0584   cp -f %(directory1)s%(director1)s_plotting.root $ALIGNMENT_CAFDIR/files/
0585   cp -f %(directory)s%(director)s_plotting.root $ALIGNMENT_CAFDIR/files/
0586   ALIGNMENT_MAPPLOTS=%(mapplots)s
0587   ALIGNMENT_SEGDIFFPLOTS=%(segdiffplots)s
0588   ALIGNMENT_CURVATUREPLOTS=%(curvatureplots)s
0589 fi
0590 
0591 dtcsc=""
0592 if [ $ALIGNMENT_DO_DT == \"True\" ]; then
0593   dtcsc="--dt"
0594 fi
0595 if [ $ALIGNMENT_DO_CSC == \"True\" ]; then
0596   dtcsc="${dtcsc} --csc"
0597 fi
0598 
0599 
0600 cd $ALIGNMENT_CAFDIR/
0601 echo \" ### Start running ###\"
0602 date
0603 
0604 # do fits and median plots first 
0605 ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out  --createDirSructure --dt --csc --fit --median
0606 
0607 if [ $ALIGNMENT_MAPPLOTS == \"True\" ]; then
0608   ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out  $dtcsc --map
0609 fi
0610 
0611 if [ $ALIGNMENT_SEGDIFFPLOTS == \"True\" ]; then
0612   ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out  $dtcsc --segdiff
0613 fi                   
0614 
0615 if [ $ALIGNMENT_CURVATUREPLOTS == \"True\" ]; then
0616   ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out  $dtcsc --curvature
0617 fi
0618 
0619 if [ $ALIGNMENT_EXTRAPLOTS == \"True\" ]; then
0620   if [ \"zzz%(copytrackerdb)s\" != \"zzz\" ]; then
0621     cp -f $ALIGNMENT_AFSDIR/%(copytrackerdb)s $ALIGNMENT_CAFDIR/
0622   fi
0623   cp $ALIGNMENT_AFSDIR/inertGlobalPositionRcd.db .
0624   ./convertSQLiteXML.py $ALIGNMENT_AFSDIR/%(INITIALGEOM)s g0.xml --noLayers  --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
0625   ./wrapperExtraPlots.sh -n $ALIGNMENT_ITERATION -i $ALIGNMENT_AFSDIR -0 g0.xml -z -w %(station123params)s %(dir_no_)s
0626   mkdir out/extra
0627   cd %(dir_no_)s
0628   mv MB ../out/extra/
0629   mv ME ../out/extra/
0630   cd -
0631 fi
0632 
0633 # run simple diagnostic
0634 ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out --dt --csc --diagnostic
0635 
0636 # fill the tree browser structure: 
0637 ./createTree.py -i $ALIGNMENT_CAFDIR/out
0638 
0639 timestamp=`date \"+%%y-%%m-%%d %%H:%%M:%%S\"`
0640 echo \"%(validationLabel)s.plots (${timestamp})\" > out/label.txt
0641 
0642 ls -l out/
0643 timestamp=`date +%%Y%%m%%d%%H%%M%%S`
0644 tar czf %(validationLabel)s_${timestamp}.tgz out
0645 cp -f %(validationLabel)s_${timestamp}.tgz $ALIGNMENT_AFSDIR/
0646 
0647 """ % my_vars)
0648 
0649 
0650 #####################################################################
0651 
0652 #SUPER_SPECIAL_XY_AND_DXDZ_ITERATIONS = True
0653 SUPER_SPECIAL_XY_AND_DXDZ_ITERATIONS = False
0654 
0655 bsubfile = ["#!/bin/sh", ""]
0656 bsubnames = []
0657 last_align = None
0658 directory = ""
0659 
0660 for iteration in range(1, ITERATIONS+1):
0661     if iteration == 1:
0662         inputdb = INITIALGEOM
0663         inputdbdir = directory[:]
0664     else:
0665         inputdb = director + ".db"
0666         inputdbdir = directory[:]
0667 
0668     directory = "%s%02d/" % (DIRNAME, iteration)
0669     director = directory[:-1]
0670 
0671     dir_no_ = DIRNAME
0672     if DIRNAME[-1]=='_': dir_no_ = DIRNAME[:-1]
0673 
0674     os.system("rm -rf %s; mkdir %s" % (directory, directory))
0675     os.system("cp Alignment/MuonAlignmentAlgorithms/python/gather_cfg.py %s" % directory)
0676     os.system("cp Alignment/MuonAlignmentAlgorithms/python/align_cfg.py %s" % directory)
0677 
0678     bsubfile.append("cd %s" % directory)
0679 
0680     mapplots = False
0681     if mapplots_ingeneral and (iteration == 1 or iteration == 3 or iteration == 5 or iteration == 7 or iteration == 9 or iteration == ITERATIONS): mapplots = True
0682     segdiffplots = False
0683     if segdiffplots_ingeneral and (iteration == 1 or iteration == ITERATIONS): segdiffplots = True
0684     curvatureplots = False
0685     if curvatureplots_ingeneral and (iteration == 1 or iteration == ITERATIONS): curvatureplots = True
0686 
0687     ### gather.sh runners for njobs
0688     for jobnumber in range(njobs):
0689         if not options.inputInBlocks:
0690             inputfiles = " ".join(fileNames[jobnumber*stepsize:(jobnumber+1)*stepsize])
0691         else:
0692             inputfiles = " ".join(fileNamesBlocks[jobnumber])
0693 
0694         if mapplots or segdiffplots or curvatureplots: copyplots = "plotting*.root"
0695         else: copyplots = ""
0696 
0697         if len(inputfiles) > 0:
0698             gather_fileName = "%sgather%03d.sh" % (directory, jobnumber)
0699             writeGatherCfg(gather_fileName, vars())
0700             os.system("chmod +x %s" % gather_fileName)
0701             bsubfile.append("echo %sgather%03d.sh" % (directory, jobnumber))
0702 
0703             if last_align is None: waiter = ""
0704             else: waiter = "-w \"ended(%s)\"" % last_align            
0705             if options.big: queue = "cmscaf1nd"
0706             else: queue = "cmscaf1nh"
0707 
0708             bsubfile.append("bsub -R \"type==SLC6_64\" -q %s -J \"%s_gather%03d\" -u youremail.tamu.edu %s gather%03d.sh" % (queue, director, jobnumber, waiter, jobnumber))
0709 
0710             bsubnames.append("ended(%s_gather%03d)" % (director, jobnumber))
0711 
0712 
0713     ### align.sh
0714     if SUPER_SPECIAL_XY_AND_DXDZ_ITERATIONS:
0715         if ( iteration == 1 or iteration == 3 or iteration == 5 or iteration == 7 or iteration == 9):
0716             tmp = station123params, station123params, useResiduals 
0717             station123params, station123params, useResiduals = "000010", "000010", "0010"
0718             writeAlignCfg("%salign.sh" % directory, vars())
0719             station123params, station123params, useResiduals = tmp
0720         elif ( iteration == 2 or iteration == 4 or iteration == 6 or iteration == 8 or iteration == 10):
0721             tmp = station123params, station123params, useResiduals 
0722             station123params, station123params, useResiduals = "110001", "100001", "1100"
0723             writeAlignCfg("%salign.sh" % directory, vars())
0724             station123params, station123params, useResiduals = tmp
0725     else:
0726         writeAlignCfg("%salign.sh" % directory, vars())
0727 
0728     os.system("chmod +x %salign.sh" % directory)
0729 
0730     bsubfile.append("echo %salign.sh" % directory)
0731     if user_mail: bsubfile.append("bsub -R \"type==SLC6_64\" -q cmscaf1nd -J \"%s_align\" -u %s -w \"%s\" align.sh" % (director, user_mail, " && ".join(bsubnames)))
0732     else: bsubfile.append("bsub -R \"type==SLC6_64\" -q cmscaf1nd -J \"%s_align\" -w \"%s\" align.sh" % (director, " && ".join(bsubnames)))
0733 
0734     #bsubfile.append("cd ..")
0735     bsubnames = []
0736     last_align = "%s_align" % director
0737 
0738 
0739     ### after the last iteration (optionally) do diagnostics run
0740     if len(validationLabel) and iteration == ITERATIONS:
0741         # do we have plotting files created?
0742         directory1 = "%s01/" % DIRNAME
0743         director1 = directory1[:-1]
0744 
0745         writeValidationCfg("%svalidation.sh" % directory, vars())
0746         os.system("chmod +x %svalidation.sh" % directory)
0747 
0748         bsubfile.append("echo %svalidation.sh" % directory)
0749         if user_mail: bsubfile.append("bsub -R \"type==SLC6_64\" -q cmscaf1nd -J \"%s_validation\" -u %s -w \"ended(%s)\" validation.sh" % (director, user_mail, last_align))
0750         else: bsubfile.append("bsub -R \"type==SLC6_64\" -q cmscaf1nd -J \"%s_validation\" -w \"ended(%s)\" validation.sh" % (director, last_align))
0751 
0752     bsubfile.append("cd ..")
0753     bsubfile.append("")
0754 
0755 
0756 file(options.submitJobs, "w").write("\n".join(bsubfile))
0757 os.system("chmod +x %s" % options.submitJobs)