Project CMSSW displayed by LXR CMSSW_14_0_X_2023-12-01-2300/​Alignment/​MillePedeAlignmentAlgorithm/​scripts/​mps_alisetup.py	Source navigation Diff markup Identifier search General search
	Version: CMSSW_14_0_X_2023-12-01-2300 CMSSW_14_0_X_2023-11-30-2300 CMSSW_14_0_X_2023-11-29-2300 CMSSW_14_0_X_2023-11-28-2300 CMSSW_14_0_X_2023-11-27-2300 CMSSW_14_0_X_2023-11-26-2300 Revert   Change

File indexing completed on 2023-10-25 09:32:17

 #!/usr/bin/env python3
 
 from __future__ import print_function
 import os
 import re
 import sys
 if sys.version_info[0]>2:
   import _pickle as cPickle
 else:
   import cPickle
 import argparse
 import itertools
 import subprocess
 import collections
 import configparser as ConfigParser
 import Alignment.MillePedeAlignmentAlgorithm.mpslib.tools as mps_tools
 import Alignment.MillePedeAlignmentAlgorithm.mpslib.Mpslibclass as mpslib
 import Alignment.MillePedeAlignmentAlgorithm.mpsvalidate.iniparser as mpsv_iniparser
 import Alignment.MillePedeAlignmentAlgorithm.mpsvalidate.trackerTree as mpsv_trackerTree
 from Alignment.MillePedeAlignmentAlgorithm.alignmentsetup.helper import checked_out_MPS
 from functools import reduce
 
 
 ################################################################################
 def main(argv = None):
     """Main routine. Not called, if this module is loaded via `import`.
 
     Arguments:
     - `argv`: Command line arguments passed to the script.
     """
 
     if argv == None:
         argv = sys.argv[1:]
 
     setup_alignment = SetupAlignment(argv)
     setup_alignment.setup()
 
 
 ################################################################################
 class SetupAlignment(object):
     """Class encapsulating the alignment campaign setup procedure."""
 
     def __init__(self, argv):
         """Constructor
 
         Arguments:
         - `argv`: command line arguments
         """
 
         self._argv = argv       # raw command line arguments
         self._args = None       # parsed command line arguments
         self._config = None     # ConfigParser object
         self._mss_dir = None    # mass storage directory
         self._datasets = collections.OrderedDict() # final dataset configs
         self._first_run = None   # first run for start geometry
         self._cms_process = None # cms.Process extracted from CMSSW config
         self._override_gt = None # snippet to append to config
         self._pede_script = None # path to pede batch script template
         self._weight_dict = collections.OrderedDict() # dictionary with dataset weights
         self._mille_script = None # path to mille batch script template
         self._mps_dir_name = None # MP campaign name (mp<ID>)
         self._common_weights = {} # dictionary with dataset weights from [weights] section
         self._weight_configs = [] # list with combinations of dataset weights
         self._general_options = {} # general options extracted from ini file
         self._external_datasets = collections.OrderedDict() # external dataset configs
         self._first_pede_config = True # does a pede job exist already?
 
         self._create_config()
         self._fill_general_options()
         self._fetch_datasets()
         self._construct_paths()
         self._create_mass_storage_directory()
         self._fetch_pede_settings()
         self._create_weight_configs()
 
 
     def setup(self):
         """Setup the alignment campaign."""
 
         if self._args.weight:
             self._create_additional_pede_jobs()
         else:
             self._create_mille_jobs()
             self._create_pede_jobs()
 
         if self._override_gt.strip() != "":
             msg = ("Overriding global tag with single-IOV tags extracted from "
                    "'{}' for run number '{}'.".format(self._global_tag,
                                                       self._first_run))
             print(msg)
             print("-"*75)
             print(self._override_gt)
             print("="*75)
 
 
     def _create_config(self):
         """Create ConfigParser object from command line arguments."""
 
         helpEpilog ="""Builds the config-templates from a universal
         config-template for each dataset specified in .ini-file that is passed
         to this script.  Then calls mps_setup.py for all datasets."""
         parser = argparse.ArgumentParser(
             description = ("Setup the alignment as configured in the "
                            "alignment_config file."),
             epilog = helpEpilog)
         parser.add_argument("-v", "--verbose", action="store_true",
                             help="display detailed output of mps_setup")
         parser.add_argument("-w", "--weight", action="store_true",
                             help=("creates additional merge job(s) with "
                                   "(possibly new) weights from .ini-config"))
         parser.add_argument("alignmentConfig",
                             help=("name of the .ini config file that specifies "
                                   "the datasets to be used"))
 
         self._args = parser.parse_args(self._argv)
         self._config = ConfigParser.ConfigParser()
         self._config.optionxform = str # default would give lowercase options
                                        # -> not wanted
         self._config.read(self._args.alignmentConfig)
         self._config.config_path = self._args.alignmentConfig
 
 
     def _construct_paths(self):
         """Determine directory paths and create the ones that are needed."""
 
         mpsTemplates = os.path.join("src", "Alignment",
                                     "MillePedeAlignmentAlgorithm", "templates")
         if checked_out_MPS()[0]:
             mpsTemplates = os.path.join(os.environ["CMSSW_BASE"], mpsTemplates)
         else:
             mpsTemplates = os.path.join(os.environ["CMSSW_RELEASE_BASE"], mpsTemplates)
         self._mille_script = os.path.join(mpsTemplates, "mps_runMille_template.sh")
         self._pede_script  = os.path.join(mpsTemplates, "mps_runPede_rfcp_template.sh")
 
         # get working directory name
         currentDir = os.getcwd()
         match = re.search(re.compile('mpproduction\/mp(.+?)$', re.M|re.I),currentDir)
         if match:
             self._mps_dir_name = 'mp'+match.group(1)
         else:
             print("Current location does not seem to be a MillePede campaign directory:", end=' ')
             print(currentDir)
             sys.exit(1)
 
 
     def _fill_general_options(self):
         """Create and fill `general_options` dictionary."""
 
         print("="*75)
         self._fetch_dataset_directory()
         self._fetch_external_datasets()
         self._fetch_essentials()
         self._fetch_defaults()
 
 
     def _fetch_external_datasets(self):
         """Fetch information about external datasets."""
 
         if self._config.has_option("general", "externalDatasets"):
             datasets = map(lambda x: x.strip(),
                            self._config.get("general",
                                             "externalDatasets").split(","))
             datasets = [x for x in datasets if len(x.strip()) > 0]
             for item in datasets:
                 splitted = item.split("|")
                 dataset = splitted[0].strip()
                 dataset = os.path.expandvars(dataset)
 
                 weight = splitted[1] if len(splitted) > 1 else None
                 config = ConfigParser.ConfigParser()
                 config.optionxform = str
                 config.read(dataset)
                 config.config_path = dataset
                 self._external_datasets[dataset] = {"config": config,
                                                     "weight": weight}
 
 
 
     def _create_mass_storage_directory(self):
         """
         Create MPS mass storage directory where, e.g., mille binaries are
         stored.
         """
 
         # set directory on eos
         self._mss_dir = self._general_options.get("massStorageDir",
                                                   "/eos/cms/store/group/alca_millepede/")
         self._mss_dir = os.path.join(self._mss_dir, "MPproductionFiles",
                                      self._mps_dir_name)
 
         cmd = ["mkdir", "-p", self._mss_dir]
 
         # create directory
         if not self._general_options.get("testMode", False):
             try:
                 with open(os.devnull, "w") as dump:
                     subprocess.check_call(cmd, stdout = dump, stderr = dump)
             except subprocess.CalledProcessError:
                 print("Failed to create mass storage directory:", self._mss_dir)
                 sys.exit(1)
 
 
     def _create_weight_configs(self):
         """Extract different weight configurations from `self._config`."""
 
         weights_list = [[(name, weight) for weight in self._weight_dict[name]]
                         for name in self._weight_dict]
 
         common_weights_list = [[(name, weight)
                                 for weight in  self._common_weights[name]]
                                for name in self._common_weights]
 
         common_weights_dicts = []
         for item in itertools.product(*common_weights_list):
             d = {}
             for name,weight in item:
                 d[name] = weight
             common_weights_dicts.append(d)
 
         weight_configs = []
         for weight_conf in itertools.product(*weights_list):
             number_of_configs = len(weight_configs)
             for common_weight in common_weights_dicts:
                 replaced_config \
                     = tuple([(dataset[0],
                               reduce(lambda x,y: mps_tools.replace_factors(x, y, common_weight[y]),
                                      common_weight, dataset[1]))
                              for dataset in weight_conf])
                 if replaced_config not in weight_configs:
                     weight_configs.append(replaced_config)
 
             # default if config contains no common weights:
             if len(weight_configs) == number_of_configs:
                 weight_configs.append(weight_conf)
 
         for weight_config in weight_configs:
             resolved_weight_config \
                 = [(dataset[0], mps_tools.compute_product_string(dataset[1]))
                    for dataset in weight_config]
             self._weight_configs.append(resolved_weight_config)
 
 
     def _fetch_pede_settings(self):
         """Fetch 'pedesettings' from general section in `self._config`."""
 
         self._pede_settings \
             = ([x.strip()
                 for x in self._config.get("general", "pedesettings").split(",")]
                if self._config.has_option("general", "pedesettings") else [None])
 
 
     def _create_mille_jobs(self):
         """Create the mille jobs based on the [dataset:<name>] sections."""
 
         gt_regex = re.compile('setupGlobaltag\s*\=\s*[\"\'](.*?)[\"\']')
         sg_regex = re.compile("setupRunStartGeometry\s*\=\s*.*$", re.M)
         collection_regex = re.compile('setupCollection\s*\=\s*[\"\'](.*?)[\"\']')
         czt_regex = re.compile('setupCosmicsZeroTesla\s*\=\s*.*$', re.M)
         cdm_regex = re.compile('setupCosmicsDecoMode\s*\=\s*.*$', re.M)
         pw_regex = re.compile('setupPrimaryWidth\s*\=\s*.*$', re.M)
         json_regex = re.compile('setupJson\s*\=\s*.*$', re.M)
 
         first_dataset = True
         for name, dataset in self._datasets.items():
             print("="*75)
             # Build config from template/Fill in variables
             try:
                 with open(dataset["configTemplate"],"r") as f:
                     tmpFile = f.read()
             except IOError:
                 print("The config-template called", end=' ')
                 print(dataset["configTemplate"], "cannot be found.")
                 sys.exit(1)
 
             tmpFile = re.sub(gt_regex,
                              'setupGlobaltag = \"'+dataset["globaltag"]+'\"',
                              tmpFile)
             tmpFile = re.sub(sg_regex,
                              "setupRunStartGeometry = "+
                              self._general_options["FirstRunForStartGeometry"], tmpFile)
             tmpFile = re.sub(collection_regex,
                              'setupCollection = \"'+dataset["collection"]+'\"',
                              tmpFile)
             if "ALCARECOTkAlCosmics" in dataset["collection"]:
                 if dataset['cosmicsZeroTesla']:
                     tmpFile = re.sub(czt_regex,
                                      'setupCosmicsZeroTesla = True',
                                      tmpFile)
                 else :
                     tmpFile = re.sub(czt_regex,
                                     'setupCosmicsZeroTesla = False',
                                      tmpFile)
 
                 if dataset['cosmicsDecoMode']:
                      tmpFile = re.sub(cdm_regex,
                                      'setupCosmicsDecoMode = True',
                                      tmpFile)
                 else:
                     tmpFile = re.sub(cdm_regex,
                                      'setupCosmicsDecoMode = False',
                                      tmpFile)
 
             if dataset['primaryWidth'] > 0.0:
                 tmpFile = re.sub(pw_regex,
                                  'setupPrimaryWidth = '+str(dataset["primaryWidth"]),
                                  tmpFile)
             if dataset['json'] != '':
                 tmpFile = re.sub(json_regex,
                                  'setupJson = \"'+dataset["json"]+'\"',
                                  tmpFile)
 
             thisCfgTemplate = "tmp.py"
             with open(thisCfgTemplate, "w") as f:
                 f.write(tmpFile)
 
 
             # Set mps_setup append option for datasets following the first one
             append = "-a"
             if first_dataset:
                 append = ""
                 first_dataset = False
                 self._config_template = tmpFile
                 self._cms_process = mps_tools.get_process_object(thisCfgTemplate)
                 self._create_input_db()
 
             with open(thisCfgTemplate, "a") as f: f.write(self._override_gt)
 
 
             # create mps_setup command
             command = ["mps_setup.py",
                        "-m",
                        append,
                        "-M", self._general_options["pedeMem"],
                        "-N", name,
                        self._mille_script,
                        thisCfgTemplate,
                        dataset["inputFileList"],
                        str(dataset["njobs"]),
                        self._general_options["classInf"],
                        self._general_options["jobname"],
                        self._pede_script,
                        "cmscafuser:"+self._mss_dir]
             if dataset["numberOfEvents"] > 0:
                 command.extend(["--max-events", str(dataset["numberOfEvents"])])
             command = [x for x in command if len(x.strip()) > 0]
 
             # Some output:
             print("Creating jobs for dataset:", name)
             print("-"*75)
             print("Baseconfig:        ", dataset["configTemplate"])
             print("Collection:        ", dataset["collection"])
             if "ALCARECOTkAlCosmics" in dataset["collection"]:
                 print("cosmicsDecoMode:   ", dataset["cosmicsDecoMode"])
                 print("cosmicsZeroTesla:  ", dataset["cosmicsZeroTesla"])
             print("Globaltag:         ", dataset["globaltag"])
             print("Number of jobs:    ", dataset["njobs"])
             print("Inputfilelist:     ", dataset["inputFileList"])
             if dataset["json"] != "":
                 print("Jsonfile:          ", dataset["json"])
             if self._args.verbose:
                 print("Pass to mps_setup: ", " ".join(command))
 
             # call the command and toggle verbose output
             self._handle_process_call(command, self._args.verbose)
 
             # remove temporary file
             self._handle_process_call(["rm", thisCfgTemplate])
 
 
     def _create_pede_jobs(self):
         """Create pede jobs from the given input."""
 
         for setting in self._pede_settings:
             print()
             print("="*75)
             if setting is None:
                 print("Creating pede job{}.".format(
                     "s" if len(self._pede_settings)*len(self._weight_configs) > 1 else ""))
                 print("-"*75)
             else:
                 print("Creating pede jobs using settings from '{0}'.".format(setting))
             for weight_conf in self._weight_configs:
                 # blank weights
                 self._handle_process_call(["mps_weight.pl", "-c"])
 
                 thisCfgTemplate = "tmp.py"
                 with open(thisCfgTemplate, "w") as f: f.write(self._config_template)
                 if self._override_gt is None:
                     self._cms_process = mps_tools.get_process_object(thisCfgTemplate)
                     self._create_input_db()
                 with open(thisCfgTemplate, "a") as f: f.write(self._override_gt)
 
                 for name,weight in weight_conf:
                     self._handle_process_call(["mps_weight.pl", "-N", name, weight], True)
 
                 if not self._first_pede_config:
                     # create new mergejob
                     self._handle_process_call(["mps_setupm.pl"], self._args.verbose)
 
                 # read mps.db to find directory of new mergejob
                 lib = mpslib.jobdatabase()
                 lib.read_db()
 
                 # short cut for jobm path
                 jobm_path = os.path.join("jobData", lib.JOBDIR[-1])
 
                 # delete old merge-config
                 command = ["rm", "-f", os.path.join(jobm_path, "alignment_merge.py")]
                 self._handle_process_call(command, self._args.verbose)
 
                 # create new merge-config
                 command = [
                     "mps_merge.py",
                     "-w", thisCfgTemplate,
                     os.path.join(jobm_path, "alignment_merge.py"),
                     jobm_path,
                     str(lib.nJobs),
                 ]
                 if setting is not None: command.extend(["-a", setting])
                 print("-"*75)
                 print(" ".join(command))
                 self._handle_process_call(command, self._args.verbose)
                 self._create_tracker_tree()
                 if self._first_pede_config:
                     os.symlink(self._tracker_tree_path,
                                os.path.abspath(os.path.join(jobm_path,
                                                             ".TrackerTree.root")))
                     self._first_pede_config = False
 
                 # store weights configuration
                 with open(os.path.join(jobm_path, ".weights.pkl"), "wb") as f:
                     cPickle.dump(weight_conf, f, 2)
                 print("="*75)
 
         # remove temporary file
         self._handle_process_call(["rm", thisCfgTemplate])
 
 
     def _create_additional_pede_jobs(self):
         """
         Create pede jobs in addition to already existing ones. Return GT
         override snippet.
         """
 
         # do some basic checks
         if not os.path.isdir("jobData"):
             print("No jobData-folder found.", end=' ')
             print("Properly set up the alignment before using the -w option.")
             sys.exit(1)
         if not os.path.exists("mps.db"):
             print("No mps.db found.", end=' ')
             print("Properly set up the alignment before using the -w option.")
             sys.exit(1)
 
         firstDataset = next(iter(self._datasets.values()))
         config_template = firstDataset["configTemplate"]
         collection = firstDataset["collection"]
 
         try:
             with open(config_template,"r") as f:
                 tmpFile = f.read()
         except IOError:
             print("The config-template '"+config_template+"' cannot be found.")
             sys.exit(1)
 
         tmpFile = re.sub('setupGlobaltag\s*\=\s*[\"\'](.*?)[\"\']',
                          'setupGlobaltag = \"'+self._global_tag+'\"',
                          tmpFile)
         tmpFile = re.sub('setupCollection\s*\=\s*[\"\'](.*?)[\"\']',
                          'setupCollection = \"'+collection+'\"',
                          tmpFile)
         tmpFile = re.sub(re.compile("setupRunStartGeometry\s*\=\s*.*$", re.M),
                          "setupRunStartGeometry = "+self._first_run,
                          tmpFile)
         self._config_template = tmpFile
 
         # first pede job exists already in this mode:
         self._first_pede_config = False
         self._create_pede_jobs()
 
 
     def _handle_process_call(self, command, verbose = False):
         """
         Wrapper around subprocess calls which treats output depending on verbosity
         level.
 
         Arguments:
         - `command`: list of command items
         - `verbose`: flag to turn on verbosity
         """
 
         call_method = subprocess.check_call if verbose else subprocess.check_output
         try:
             call_method(command, stderr=subprocess.STDOUT)
         except subprocess.CalledProcessError as e:
             print("" if verbose else e.output)
             print("Failed to execute command:", " ".join(command))
             sys.exit(1)
 
 
     def _create_input_db(self):
         """
         Create sqlite file with single-IOV tags and use it to override the
         GT. If the GT is already customized by the user, the customization has
         higher priority. Creates a snippet to be appended to the configuration
         file.
         """
 
         run_number = int(self._first_run)
         if not run_number > 0:
             print("'FirstRunForStartGeometry' must be positive, but is", run_number)
             sys.exit(1)
 
         input_db_name = os.path.abspath("alignment_input.db")
         tags = mps_tools.create_single_iov_db(self._check_iov_definition(),
                                               run_number, input_db_name)
 
         self._override_gt = ""
         for record,tag in tags.items():
             if self._override_gt == "":
                 self._override_gt \
                     += ("\nimport "
                         "Alignment.MillePedeAlignmentAlgorithm.alignmentsetup."
                         "SetCondition as tagwriter\n")
             self._override_gt += ("\ntagwriter.setCondition(process,\n"
                                   "       connect = \""+tag["connect"]+"\",\n"
                                   "       record = \""+record+"\",\n"
                                   "       tag = \""+tag["tag"]+"\")\n")
 
 
     def _check_iov_definition(self):
         """
         Check consistency of input alignment payloads and IOV definition.
         Returns a dictionary with the information needed to override possibly
         problematic input taken from the global tag.
         """
 
         print("Checking consistency of IOV definition...")
         iovs = mps_tools.make_unique_runranges(self._cms_process.AlignmentProducer)
 
         inputs = {
             "TrackerAlignmentRcd": None,
             "TrackerSurfaceDeformationRcd": None,
             "TrackerAlignmentErrorExtendedRcd": None,
         }
 
         for condition in self._cms_process.GlobalTag.toGet.value():
             if condition.record.value() in inputs:
                 inputs[condition.record.value()] = {
                     "tag": condition.tag.value(),
                     "connect": ("pro"
                                 if not condition.hasParameter("connect")
                                 else condition.connect.value())
                 }
 
         inputs_from_gt = [record for record in inputs if inputs[record] is None]
         inputs.update(
             mps_tools.get_tags(self._cms_process.GlobalTag.globaltag.value(),
                                inputs_from_gt))
 
         if int(self._first_run) != iovs[0]:     # simple consistency check
             if iovs[0] == 1 and len(iovs) == 1:
                 print("Single IOV output detected in configuration and", end=' ')
                 print("'FirstRunForStartGeometry' is not 1.")
                 print("Creating single IOV output from input conditions in run", end=' ')
                 print(self._first_run+".")
                 for inp in inputs: inputs[inp]["problematic"] = True
             else:
                 print("Value of 'FirstRunForStartGeometry' has to match first", end=' ')
                 print("defined output IOV:", end=' ')
                 print(self._first_run, "!=", iovs[0])
                 sys.exit(1)
 
         for inp in inputs.values():
             inp["iovs"] = mps_tools.get_iovs(inp["connect"], inp["tag"])
 
         # check consistency of input with output
         problematic_gt_inputs = {}
         input_indices = {key: len(value["iovs"]) -1
                          for key,value in inputs.items()}
         for iov in reversed(iovs):
             for inp in inputs:
                 if inputs[inp].pop("problematic", False):
                     problematic_gt_inputs[inp] = inputs[inp]
                 if inp in problematic_gt_inputs: continue
                 if input_indices[inp] < 0:
                     print("First output IOV boundary at run", iov, end=' ')
                     print("is before the first input IOV boundary at", end=' ')
                     print(inputs[inp]["iovs"][0], "for '"+inp+"'.")
                     print("Please check your run range selection.")
                     sys.exit(1)
                 input_iov = inputs[inp]["iovs"][input_indices[inp]]
                 if iov < input_iov:
                     if inp in inputs_from_gt:
                         problematic_gt_inputs[inp] = inputs[inp]
                         print("Found problematic input taken from global tag.")
                         print("Input IOV boundary at run",input_iov, end=' ')
                         print("for '"+inp+"' is within output IOV starting", end=' ')
                         print("with run", str(iov)+".")
                         print("Deriving an alignment with coarse IOV", end=' ')
                         print("granularity starting from finer granularity", end=' ')
                         print("leads to wrong results.")
                         print("A single IOV input using the IOV of", end=' ')
                         print("'FirstRunForStartGeometry' ("+self._first_run+")", end=' ')
                         print("is automatically created and used.")
                         continue
                     print("Found input IOV boundary at run",input_iov, end=' ')
                     print("for '"+inp+"' which is within output IOV starting", end=' ')
                     print("with run", str(iov)+".")
                     print("Deriving an alignment with coarse IOV granularity", end=' ')
                     print("starting from finer granularity leads to wrong", end=' ')
                     print("results.")
                     print("Please check your run range selection.")
                     sys.exit(1)
                 elif iov == input_iov:
                     input_indices[inp] -= 1
 
         # check consistency of 'TrackerAlignmentRcd' with other inputs
         input_indices = {key: len(value["iovs"]) -1
                          for key,value in inputs.items()
                          if (key != "TrackerAlignmentRcd")
                          and (inp not in problematic_gt_inputs)}
         for iov in reversed(inputs["TrackerAlignmentRcd"]["iovs"]):
             for inp in input_indices:
                 input_iov = inputs[inp]["iovs"][input_indices[inp]]
                 if iov < input_iov:
                     print("Found input IOV boundary at run",input_iov, end=' ')
                     print("for '"+inp+"' which is within 'TrackerAlignmentRcd'", end=' ')
                     print("IOV starting with run", str(iov)+".")
                     print("Deriving an alignment with inconsistent IOV boundaries", end=' ')
                     print("leads to wrong results.")
                     print("Please check your input IOVs.")
                     sys.exit(1)
                 elif iov == input_iov:
                     input_indices[inp] -= 1
 
         print(" -> IOV consistency check successful.")
         print("="*75)
 
         return problematic_gt_inputs
 
 
     def _create_tracker_tree(self):
         """Method to create hidden 'TrackerTree.root'."""
 
         if self._global_tag is None or self._first_run is None:
             print("Trying to create the tracker tree before setting the global", end=' ')
             print("tag or the run to determine the geometry IOV.")
             sys.exit(1)
 
         config = mpsv_iniparser.ConfigData()
         config.jobDataPath = "."    # current directory
         config.globalTag = self._global_tag
         config.firstRun = self._first_run
         self._tracker_tree_path = mpsv_trackerTree.check(config)
 
 
     def _fetch_essentials(self):
         """Fetch general options from config file."""
 
         for var in ("classInf","pedeMem","jobname", "FirstRunForStartGeometry"):
             try:
                 self._general_options[var] = self._config.get('general',var)
             except ConfigParser.NoOptionError:
                 print("No", var, "found in [general] section.", end=' ')
                 print("Please check ini-file.")
                 sys.exit(1)
         self._first_run = self._general_options["FirstRunForStartGeometry"]
 
 
     def _fetch_defaults(self):
         """Fetch default general options from config file."""
 
         for var in ("globaltag", "configTemplate", "json", "massStorageDir",
                     "testMode"):
             try:
                 self._general_options[var] = self._config.get("general", var)
             except ConfigParser.NoOptionError:
                 if var == "testMode": continue
                 print("No '" + var + "' given in [general] section.")
 
         for dataset in self._external_datasets.values():
             dataset["general"] = {}
             for var in ("globaltag", "configTemplate", "json"):
                 try:
                     dataset["general"][var] = dataset["config"].get("general", var)
                 except (ConfigParser.NoSectionError,ConfigParser.NoOptionError):
                     pass
 
 
     def _fetch_dataset_directory(self):
         """
         Fetch 'datasetDir' variable from general section and add it to the
         'os.environ' dictionary.
         """
 
         if self._config.has_option("general", "datasetdir"):
             dataset_directory = self._config.get("general", "datasetdir")
             # add it to environment for later variable expansion:
             os.environ["datasetdir"] = dataset_directory
             self._general_options["datasetdir"] = dataset_directory
         else:
             print("No datasetdir given in [general] section.", end=' ')
             print("Be sure to give a full path in inputFileList.")
             self._general_options["datasetdir"] = ""
 
 
     def _fetch_datasets(self):
         """Fetch internal and external dataset configurations."""
 
         all_configs = collections.OrderedDict()
         all_configs["main"] = {"config": self._config,
                                "general": self._general_options,
                                "weight": None}
         all_configs.update(self._external_datasets)
 
         for config in all_configs.values():
             global_weight = "1" if config["weight"] is None else config["weight"]
             if global_weight+self._config.config_path in self._common_weights:
                 global_weight = self._common_weights[global_weight+
                                                      self._config.config_path]
             elif global_weight in self._common_weights:
                 global_weight = self._common_weights[global_weight]
             else:
                 global_weight = (global_weight,)
             common_weights = {}
             weight_dict = {}
             for section in config["config"].sections():
                 cache_datasetdir = os.environ["datasetdir"]
                 if "general" in section:
                     if config["config"].has_option("general", "datasetdir"):
                         os.environ["datasetdir"] = config["config"].get("general", "datasetdir")
                 elif section == "weights":
                     for option in config["config"].options(section):
                         common_weights[option] \
                             = [x.strip() for x in
                                config["config"].get(section, option).split(",")]
                 elif section.startswith("dataset:"):
                     print("-"*75)
                     # set name from section-name
                     name = section[8:]
                     if name in self._datasets:
                         print("WARNING: Duplicate definition of dataset '{}'".format(name))
                         print(" -> Using defintion in '{}':\n".format(config["config"].config_path))
                         print("    [{}]".format(section))
                         for k,v in config["config"].items(section):
                             print("   ", k, "=", v)
                         print()
                     self._datasets[name] = {}
 
                     # extract weight for the dataset
                     if config["config"].has_option(section, "weight"):
                         self._weight_dict[name] \
                             = [x.strip() for x in
                                config["config"].get(section, "weight").split(",")]
                     else:
                         self._weight_dict[name] = ["1.0"]
                     self._weight_dict[name] = [global_w+"*"+w
                                                for w in self._weight_dict[name]
                                                for global_w in global_weight]
                     weight_dict[name] = self._weight_dict[name]
 
                     # extract essential variables
                     for var in ("inputFileList", "collection"):
                         try:
                             self._datasets[name][var] = config["config"].get(section, var)
                         except ConfigParser.NoOptionError:
                             print("No", var, "found in", section+". Please check ini-file.")
                             sys.exit(1)
 
                     # get globaltag and configTemplate. If none in section, try to get
                     # default from [general] section.
                     for var in ("configTemplate", "globaltag"):
                         try:
                             self._datasets[name][var] = config["config"].get(section, var)
                         except (ConfigParser.NoSectionError,ConfigParser.NoOptionError):
                             try:
                                 self._datasets[name][var] = config["general"][var]
                             except KeyError:
                                 try:
                                     self._datasets[name][var] \
                                         = all_configs["main"]["general"][var]
                                 except KeyError:
                                     print("No",var,"found in ["+section+"]", end=' ')
                                     print("and no default in [general] section.")
                                     sys.exit(1)
 
                     # extract non-essential options
                     if "ALCARECOTkAlCosmics" in self._datasets[name]["collection"]:
                         try:
                             self._datasets[name]["cosmicsZeroTesla"] \
                                 = config["config"].getboolean(section,"cosmicsZeroTesla")
                         except ConfigParser.NoOptionError:
                             print("No option cosmicsZeroTesla found in", section,"even though it is required for dataset type", self._datasets[name]["collection"], ". Please check ini-file.")
                             sys.exit(1)
                         try:
                             self._datasets[name]["cosmicsDecoMode"] \
                                 = config["config"].getboolean(section,"cosmicsDecoMode")
                         except ConfigParser.NoOptionError:
                             print("No option cosmicsDecoMode found in", section,"even though it is required for dataset type", self._datasets[name]["collection"], ".Please check ini-file.")
                             sys.exit(1)
 
                     self._datasets[name]["primaryWidth"] = -1.0
                     if config["config"].has_option(section,"primaryWidth"):
                         self._datasets[name]["primaryWidth"] \
                             = config["config"].getfloat(section,"primaryWidth")
 
                     self._datasets[name]["numberOfEvents"] = -1
                     if config["config"].has_option(section, "numberOfEvents"):
                         self._datasets[name]["numberOfEvents"] \
                             = config["config"].getint(section, "numberOfEvents")
 
                     self._datasets[name]["json"] = ""
                     try:
                         self._datasets[name]["json"] = config["config"].get(section,"json")
                     except ConfigParser.NoOptionError:
                         try:
                             self._datasets[name]["json"] = config["general"]["json"]
                         except KeyError:
                             try:
                                 self._datasets[name]["json"] \
                                     = all_configs["main"]["general"]["json"]
                             except KeyError:
                                 print("No json given in either [general] or", end=' ')
                                 print("["+section+"] sections.")
                                 print(" -> Proceeding without json-file.")
 
 
                     #replace ${datasetdir} and other variables, e.g. $CMSSW_BASE
                     for var in ("inputFileList", "json", "configTemplate"):
                         self._datasets[name][var] \
                             = os.path.expandvars(self._datasets[name][var])
 
 
                     # Get number of jobs from lines in inputfilelist
                     self._datasets[name]["njobs"] = 0
                     try:
                         with open(self._datasets[name]["inputFileList"], "r") as filelist:
                             for line in filelist:
                                 if "CastorPool" in line:
                                     continue
                                 # ignore empty lines
                                 if not line.strip()=="":
                                     self._datasets[name]["njobs"] += 1
                     except IOError:
                         print("Inputfilelist", self._datasets[name]["inputFileList"], end=' ')
                         print("does not exist.")
                         sys.exit(1)
                     if self._datasets[name]["njobs"] == 0:
                         print("Number of jobs is 0. There may be a problem with the inputfilelist:")
                         print(self._datasets[name]["inputFileList"])
                         sys.exit(1)
 
                     # Check if njobs gets overwritten in .ini-file
                     if config["config"].has_option(section, "njobs"):
                         if config["config"].getint(section, "njobs") <= self._datasets[name]["njobs"]:
                             self._datasets[name]["njobs"] = config["config"].getint(section, "njobs")
                         else:
                             print("'njobs' is bigger than the number of files for this", end=' ')
                             print("dataset:", self._datasets[name]["njobs"])
                             print("Using default.")
                     else:
                         print("No number of jobs specified. Using number of files in", end=' ')
                         print("inputfilelist as the number of jobs.")
 
             # check if local weights override global weights and resolve name clashes
             for weight_name, weight_values in common_weights.items():
                 for key, weight in weight_dict.items():
                     if any([weight_name in w for w in weight]):
                         self._common_weights[weight_name+config["config"].config_path] = weight_values
                         self._weight_dict[key] = [mps_tools.replace_factors(w,
                                                                             weight_name,
                                                                             weight_name+config["config"].config_path)
                                                   for w in weight]
                     else:
                         self._common_weights[weight_name] = weight_values
                         self._weight_dict[key] = weight
 
             os.environ["datasetdir"] = cache_datasetdir
 
         if len(self._datasets) == 0:
             print("No dataset section defined in '{0}'".format(
                 ", ".join([self._args.aligmentConfig]+self._external_datasets.keys())))
             print("At least one section '[dataset:<name>]' is required.")
             sys.exit(1)
 
         self._global_tag = self._datasets[name]["globaltag"]
 
 
 ################################################################################
 if __name__ == "__main__":
     try:
         main()
     except KeyboardInterrupt:
         pass

This page was automatically generated by the 2.2.1 LXR engine. The LXR team