Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2021-07-23 02:25:13

0001 from __future__ import print_function
0002 from builtins import range
0003 import os
0004 import re
0005 import sys
0006 import shutil
0007 import importlib
0008 import sqlalchemy
0009 import subprocess
0010 import CondCore.Utilities.conddblib as conddb
0011 from functools import reduce
0012 
0013 
0014 def create_single_iov_db(inputs, run_number, output_db):
0015     """Create an sqlite file with single-IOV tags for alignment payloads.
0016 
0017     Arguments:
0018     - `inputs`: dictionary with input needed for payload extraction
0019     - `run_number`: run for which the IOVs are selected
0020     - `output_db`: name of the output sqlite file
0021     """
0022 
0023     # find the IOV containing `run_number`
0024     for record,tag in inputs.items():
0025         run_is_covered = False
0026         for iov in reversed(tag["iovs"]):
0027             if iov <= run_number:
0028                 tag["since"] = str(iov)
0029                 run_is_covered = True
0030                 break
0031         if not run_is_covered:
0032             msg = ("Run number {0:d} is not covered in '{1:s}' ({2:s}) from"
0033                    " '{3:s}'.".format(run_number, tag["tag"], record,
0034                                       global_tag))
0035             print(msg)
0036             print("Aborting...")
0037             sys.exit(1)
0038 
0039     result = {}
0040     remove_existing_object(output_db)
0041 
0042     for record,tag in inputs.items():
0043         result[record] = {"connect": "sqlite_file:"+output_db,
0044                           "tag": "_".join([tag["tag"], tag["since"]])}
0045 
0046         if tag["connect"] == "pro":
0047             source_connect = "frontier://FrontierProd/CMS_CONDITIONS"
0048         elif tag["connect"] == "dev":
0049             source_connect = "frontier://FrontierPrep/CMS_CONDITIONS"
0050         else:
0051             source_connect = tag["connect"]
0052 
0053         cmd = ("conddb_import",
0054                "-f", source_connect,
0055                "-c", result[record]["connect"],
0056                "-i", tag["tag"],
0057                "-t", result[record]["tag"],
0058                "-b", str(run_number),
0059                "-e", str(run_number))
0060         run_checked(cmd)
0061     if len(inputs) > 0:
0062         run_checked(["sqlite3", output_db, "update iov set since=1"])
0063 
0064     return result
0065 
0066 
0067 def run_checked(cmd, suppress_stderr = False):
0068     """Run `cmd` and exit in case of failures.
0069 
0070     Arguments:
0071     - `cmd`: list containing the strings of the command
0072     - `suppress_stderr`: suppress output from stderr
0073     """
0074 
0075     try:
0076         with open(os.devnull, "w") as devnull:
0077             if suppress_stderr:
0078                 subprocess.check_call(cmd, stdout = devnull, stderr = devnull)
0079             else:
0080                 subprocess.check_call(cmd, stdout = devnull)
0081     except subprocess.CalledProcessError as e:
0082         print("Problem in running the following command:")
0083         print(" ".join(e.cmd))
0084         sys.exit(1)
0085 
0086 
0087 def get_process_object(cfg):
0088     """Returns cms.Process object defined in `cfg`.
0089 
0090     Arguments:
0091     - `cfg`: path to CMSSW config file
0092     """
0093 
0094     sys.path.append(os.path.dirname(cfg)) # add location to python path
0095     cache_stdout = sys.stdout
0096     sys.stdout = open(os.devnull, "w")    # suppress unwanted output
0097     try:
0098         __configuration = \
0099             importlib.import_module(os.path.splitext(os.path.basename(cfg))[0])
0100     except Exception as e:
0101         print("Problem detected in configuration file '{0}'.".format(cfg))
0102         raise e
0103     sys.stdout = cache_stdout
0104     sys.path.pop()                        # clean up python path again
0105     try:
0106         os.remove(cfg+"c")                # try to remove temporary .pyc file
0107     except OSError as e:
0108         if e.args == (2, "No such file or directory"): pass
0109         else: raise
0110 
0111     return __configuration.process
0112 
0113 
0114 def make_unique_runranges(ali_producer):
0115     """Derive unique run ranges from AlignmentProducer PSet.
0116 
0117     Arguments:
0118     - `ali_producer`: cms.PSet containing AlignmentProducer configuration
0119     """
0120 
0121     if (hasattr(ali_producer, "RunRangeSelection") and
0122         len(ali_producer.RunRangeSelection) > 0):
0123         iovs = set([int(iov)
0124                     for sel in ali_producer.RunRangeSelection
0125                     for iov in sel.RunRanges])
0126         if len(iovs) == 0: return [1] # single IOV starting from run 1
0127         return sorted(iovs)
0128     else:
0129         return [1]                    # single IOV starting from run 1
0130 
0131 
0132 def get_tags(global_tag, records):
0133     """Get tags for `records` contained in `global_tag`.
0134 
0135     Arguments:
0136     - `global_tag`: global tag of interest
0137     - `records`: database records of interest
0138     """
0139 
0140     if len(records) == 0: return {} # avoid useless DB query
0141 
0142     # check for auto GT
0143     if global_tag.startswith("auto:"):
0144         import Configuration.AlCa.autoCond as AC
0145         try:
0146             global_tag = AC.autoCond[global_tag.split("auto:")[-1]]
0147         except KeyError:
0148             print("Unsupported auto GT:", global_tag)
0149             sys.exit(1)
0150 
0151     # setting up the DB session
0152     con = conddb.connect(url = conddb.make_url())
0153     session = con.session()
0154     GlobalTagMap = session.get_dbtype(conddb.GlobalTagMap)
0155 
0156     # query tag names for records of interest contained in `global_tag`
0157     tags = session.query(GlobalTagMap.record, GlobalTagMap.tag_name).\
0158            filter(GlobalTagMap.global_tag_name == global_tag,
0159                   GlobalTagMap.record.in_(records)).all()
0160 
0161     # closing the DB session
0162     session.close()
0163 
0164     return {item[0]: {"tag": item[1], "connect": "pro"} for item in tags}
0165 
0166 
0167 def get_iovs(db, tag):
0168     """Retrieve the list of IOVs from `db` for `tag`.
0169 
0170     Arguments:
0171     - `db`: database connection string
0172     - `tag`: tag of database record
0173     """
0174 
0175     db = db.replace("sqlite_file:", "").replace("sqlite:", "")
0176     db = db.replace("frontier://FrontierProd/CMS_CONDITIONS", "pro")
0177     db = db.replace("frontier://FrontierPrep/CMS_CONDITIONS", "dev")
0178 
0179     con = conddb.connect(url = conddb.make_url(db))
0180     session = con.session()
0181     IOV = session.get_dbtype(conddb.IOV)
0182 
0183     iovs = set(session.query(IOV.since).filter(IOV.tag_name == tag).all())
0184     if len(iovs) == 0:
0185         print("No IOVs found for tag '"+tag+"' in database '"+db+"'.")
0186         sys.exit(1)
0187 
0188     session.close()
0189 
0190     return sorted([int(item[0]) for item in iovs])
0191 
0192 
0193 def replace_factors(product_string, name, value):
0194     """Takes a `product_string` and replaces all factors with `name` by `value`.
0195 
0196     Arguments:
0197     - `product_string`: input string containing a product
0198     - `name`: name of the factor
0199     - `value`: value of the factor
0200     """
0201 
0202     value = str(value)                                 # ensure it's a string
0203     return re.sub(r"^"+name+r"$", value,               # single factor
0204                   re.sub(r"[*]"+name+r"$", r"*"+value, # rhs
0205                          re.sub(r"^"+name+r"[*]", value+r"*", # lhs
0206                                 re.sub(r"[*]"+name+r"[*]", r"*"+value+r"*",
0207                                        product_string))))
0208 
0209 def compute_product_string(product_string):
0210     """Takes `product_string` and returns the product of the factors as string.
0211 
0212     Arguments:
0213     - `product_string`: string containing product ('<factor>*<factor>*...')
0214     """
0215 
0216     factors = [float(f) for f in product_string.split("*")]
0217     return str(reduce(lambda x,y: x*y, factors))
0218 
0219 
0220 def check_proxy():
0221     """Check if GRID proxy has been initialized."""
0222 
0223     try:
0224         with open(os.devnull, "w") as dump:
0225             subprocess.check_call(["voms-proxy-info", "--exists"],
0226                                   stdout = dump, stderr = dump)
0227     except subprocess.CalledProcessError:
0228         return False
0229     return True
0230 
0231 
0232 def remove_existing_object(path):
0233     """
0234     Tries to remove file or directory located at `path`. If the user
0235     has no delete permissions, the object is moved to a backup
0236     file. If this fails it tries 5 times in total and then asks to
0237     perform a cleanup by a user with delete permissions.
0238 
0239     Arguments:
0240     - `name`: name of the object to be (re)moved
0241     """
0242 
0243     if os.path.exists(path):
0244         remove_method = shutil.rmtree if os.path.isdir(path) else os.remove
0245         move_method = shutil.move if os.path.isdir(path) else os.rename
0246         try:
0247             remove_method(path)
0248         except OSError as e:
0249             if e.args != (13, "Permission denied"): raise
0250             backup_path = path.rstrip("/")+"~"
0251             for _ in range(5):
0252                 try:
0253                     if os.path.exists(backup_path): remove_method(backup_path)
0254                     move_method(path, backup_path)
0255                     break
0256                 except OSError as e:
0257                     if e.args != (13, "Permission denied"): raise
0258                     backup_path += "~"
0259         if os.path.exists(path):
0260             msg = ("Cannot remove '{}' due to missing 'delete' ".format(path)
0261                    +"permissions and the limit of 5 backups is reached. Please "
0262                    "ask a user with 'delete' permissions to clean up.")
0263             print(msg)
0264             sys.exit(1)