python/mpslib/tools.py

0001 from __future__ import print_function
0002 from builtins import range
0003 import os
0004 import re
0005 import sys
0006 import shutil
0007 import importlib
0008 import sqlalchemy
0009 import subprocess
0010 import CondCore.Utilities.conddblib as conddb
0011 from functools import reduce
0012
0013
0014 def create_single_iov_db(inputs, run_number, output_db):
0015     """Create an sqlite file with single-IOV tags for alignment payloads.
0016
0017     Arguments:
0018     - `inputs`: dictionary with input needed for payload extraction
0019     - `run_number`: run for which the IOVs are selected
0020     - `output_db`: name of the output sqlite file
0021     """
0022
0023     # find the IOV containing `run_number`
0024     for record,tag in inputs.items():
0025         run_is_covered = False
0026         for iov in reversed(tag["iovs"]):
0027             if iov <= run_number:
0028                 tag["since"] = str(iov)
0029                 run_is_covered = True
0030                 break
0031         if not run_is_covered:
0032             msg = ("Run number {0:d} is not covered in '{1:s}' ({2:s}) from"
0033                    " '{3:s}'.".format(run_number, tag["tag"], record,
0034                                       global_tag))
0035             print(msg)
0036             print("Aborting...")
0037             sys.exit(1)
0038
0039     result = {}
0040     remove_existing_object(output_db)
0041
0042     for record,tag in inputs.items():
0043         result[record] = {"connect": "sqlite_file:"+output_db,
0044                           "tag": "_".join([tag["tag"], tag["since"]])}
0045
0046         if tag["connect"] == "pro":
0047             source_connect = "frontier://FrontierProd/CMS_CONDITIONS"
0048         elif tag["connect"] == "dev":
0049             source_connect = "frontier://FrontierPrep/CMS_CONDITIONS"
0050         else:
0051             source_connect = tag["connect"]
0052
0053         cmd = ("conddb_import",
0054                "-f", source_connect,
0055                "-c", result[record]["connect"],
0056                "-i", tag["tag"],
0057                "-t", result[record]["tag"],
0058                "-b", str(run_number),
0059                "-e", str(run_number))
0060         run_checked(cmd)
0061     if len(inputs) > 0:
0062         run_checked(["sqlite3", output_db, "update iov set since=1"])
0063
0064     return result
0065
0066
0067 def run_checked(cmd, suppress_stderr = False):
0068     """Run `cmd` and exit in case of failures.
0069
0070     Arguments:
0071     - `cmd`: list containing the strings of the command
0072     - `suppress_stderr`: suppress output from stderr
0073     """
0074
0075     try:
0076         with open(os.devnull, "w") as devnull:
0077             if suppress_stderr:
0078                 subprocess.check_call(cmd, stdout = devnull, stderr = devnull)
0079             else:
0080                 subprocess.check_call(cmd, stdout = devnull)
0081     except subprocess.CalledProcessError as e:
0082         print("Problem in running the following command:")
0083         print(" ".join(e.cmd))
0084         sys.exit(1)
0085
0086
0087 def get_process_object(cfg):
0088     """Returns cms.Process object defined in `cfg`.
0089
0090     Arguments:
0091     - `cfg`: path to CMSSW config file
0092     """
0093
0094     sys.path.append(os.path.dirname(cfg)) # add location to python path
0095     cache_stdout = sys.stdout
0096     sys.stdout = open(os.devnull, "w")    # suppress unwanted output
0097     try:
0098         __configuration = \
0099             importlib.import_module(os.path.splitext(os.path.basename(cfg))[0])
0100     except Exception as e:
0101         print("Problem detected in configuration file '{0}'.".format(cfg))
0102         raise e
0103     sys.stdout = cache_stdout
0104     sys.path.pop()                        # clean up python path again
0105     try:
0106         os.remove(cfg+"c")                # try to remove temporary .pyc file
0107     except OSError as e:
0108         if e.args == (2, "No such file or directory"): pass
0109         else: raise
0110
0111     return __configuration.process
0112
0113
0114 def make_unique_runranges(ali_producer):
0115     """Derive unique run ranges from AlignmentProducer PSet.
0116
0117     Arguments:
0118     - `ali_producer`: cms.PSet containing AlignmentProducer configuration
0119     """
0120
0121     if (hasattr(ali_producer, "RunRangeSelection") and
0122         len(ali_producer.RunRangeSelection) > 0):
0123         iovs = set([int(iov)
0124                     for sel in ali_producer.RunRangeSelection
0125                     for iov in sel.RunRanges])
0126         if len(iovs) == 0: return [1] # single IOV starting from run 1
0127         return sorted(iovs)
0128     else:
0129         return [1]                    # single IOV starting from run 1
0130
0131
0132 def get_tags(global_tag, records):
0133     """Get tags for `records` contained in `global_tag`.
0134
0135     Arguments:
0136     - `global_tag`: global tag of interest
0137     - `records`: database records of interest
0138     """
0139
0140     if len(records) == 0: return {} # avoid useless DB query
0141
0142     # check for auto GT
0143     if global_tag.startswith("auto:"):
0144         import Configuration.AlCa.autoCond as AC
0145         try:
0146             global_tag = AC.autoCond[global_tag.split("auto:")[-1]]
0147         except KeyError:
0148             print("Unsupported auto GT:", global_tag)
0149             sys.exit(1)
0150
0151     # setting up the DB session
0152     con = conddb.connect(url = conddb.make_url())
0153     session = con.session()
0154     GlobalTagMap = session.get_dbtype(conddb.GlobalTagMap)
0155
0156     # query tag names for records of interest contained in `global_tag`
0157     tags = session.query(GlobalTagMap.record, GlobalTagMap.tag_name).\
0158            filter(GlobalTagMap.global_tag_name == global_tag,
0159                   GlobalTagMap.record.in_(records)).all()
0160
0161     # closing the DB session
0162     session.close()
0163
0164     return {item[0]: {"tag": item[1], "connect": "pro"} for item in tags}
0165
0166
0167 def get_iovs(db, tag):
0168     """Retrieve the list of IOVs from `db` for `tag`.
0169
0170     Arguments:
0171     - `db`: database connection string
0172     - `tag`: tag of database record
0173     """
0174
0175     db = db.replace("sqlite_file:", "").replace("sqlite:", "")
0176     db = db.replace("frontier://FrontierProd/CMS_CONDITIONS", "pro")
0177     db = db.replace("frontier://FrontierPrep/CMS_CONDITIONS", "dev")
0178
0179     con = conddb.connect(url = conddb.make_url(db))
0180     session = con.session()
0181     IOV = session.get_dbtype(conddb.IOV)
0182
0183     iovs = set(session.query(IOV.since).filter(IOV.tag_name == tag).all())
0184     if len(iovs) == 0:
0185         print("No IOVs found for tag '"+tag+"' in database '"+db+"'.")
0186         sys.exit(1)
0187
0188     session.close()
0189
0190     return sorted([int(item[0]) for item in iovs])
0191
0192
0193 def replace_factors(product_string, name, value):
0194     """Takes a `product_string` and replaces all factors with `name` by `value`.
0195
0196     Arguments:
0197     - `product_string`: input string containing a product
0198     - `name`: name of the factor
0199     - `value`: value of the factor
0200     """
0201
0202     value = str(value)                                 # ensure it's a string
0203     return re.sub(r"^"+name+r"$", value,               # single factor
0204                   re.sub(r"[*]"+name+r"$", r"*"+value, # rhs
0205                          re.sub(r"^"+name+r"[*]", value+r"*", # lhs
0206                                 re.sub(r"[*]"+name+r"[*]", r"*"+value+r"*",
0207                                        product_string))))
0208
0209 def compute_product_string(product_string):
0210     """Takes `product_string` and returns the product of the factors as string.
0211
0212     Arguments:
0213     - `product_string`: string containing product ('<factor>*<factor>*...')
0214     """
0215
0216     factors = [float(f) for f in product_string.split("*")]
0217     return str(reduce(lambda x,y: x*y, factors))
0218
0219
0220 def check_proxy():
0221     """Check if GRID proxy has been initialized."""
0222
0223     try:
0224         with open(os.devnull, "w") as dump:
0225             subprocess.check_call(["voms-proxy-info", "--exists"],
0226                                   stdout = dump, stderr = dump)
0227     except subprocess.CalledProcessError:
0228         return False
0229     return True
0230
0231
0232 def remove_existing_object(path):
0233     """
0234     Tries to remove file or directory located at `path`. If the user
0235     has no delete permissions, the object is moved to a backup
0236     file. If this fails it tries 5 times in total and then asks to
0237     perform a cleanup by a user with delete permissions.
0238
0239     Arguments:
0240     - `name`: name of the object to be (re)moved
0241     """
0242
0243     if os.path.exists(path):
0244         remove_method = shutil.rmtree if os.path.isdir(path) else os.remove
0245         move_method = shutil.move if os.path.isdir(path) else os.rename
0246         try:
0247             remove_method(path)
0248         except OSError as e:
0249             if e.args != (13, "Permission denied"): raise
0250             backup_path = path.rstrip("/")+"~"
0251             for _ in range(5):
0252                 try:
0253                     if os.path.exists(backup_path): remove_method(backup_path)
0254                     move_method(path, backup_path)
0255                     break
0256                 except OSError as e:
0257                     if e.args != (13, "Permission denied"): raise
0258                     backup_path += "~"
0259         if os.path.exists(path):
0260             msg = ("Cannot remove '{}' due to missing 'delete' ".format(path)
0261                    +"permissions and the limit of 5 backups is reached. Please "
0262                    "ask a user with 'delete' permissions to clean up.")
0263             print(msg)
0264             sys.exit(1)