Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2024-11-25 02:29:03

0001 from builtins import range
0002 import os
0003 import re
0004 import sys
0005 import shutil
0006 import importlib
0007 import sqlalchemy
0008 import subprocess
0009 import CondCore.Utilities.conddblib as conddb
0010 from functools import reduce
0011 
0012 
0013 def create_single_iov_db(inputs, run_number, output_db):
0014     """Create an sqlite file with single-IOV tags for alignment payloads.
0015 
0016     Arguments:
0017     - `inputs`: dictionary with input needed for payload extraction
0018     - `run_number`: run for which the IOVs are selected
0019     - `output_db`: name of the output sqlite file
0020     """
0021 
0022     # find the IOV containing `run_number`
0023     for record,tag in inputs.items():
0024         run_is_covered = False
0025         for iov in reversed(tag["iovs"]):
0026             if iov <= run_number:
0027                 tag["since"] = str(iov)
0028                 run_is_covered = True
0029                 break
0030         if not run_is_covered:
0031             msg = ("Run number {0:d} is not covered in '{1:s}' ({2:s}) from"
0032                    " '{3:s}'.".format(run_number, tag["tag"], record,
0033                                       global_tag))
0034             print(msg)
0035             print("Aborting...")
0036             sys.exit(1)
0037 
0038     result = {}
0039     remove_existing_object(output_db)
0040 
0041     for record,tag in inputs.items():
0042         result[record] = {"connect": "sqlite_file:"+output_db,
0043                           "tag": "_".join([tag["tag"], tag["since"]])}
0044 
0045         if tag["connect"] == "pro":
0046             source_connect = "frontier://FrontierProd/CMS_CONDITIONS"
0047         elif tag["connect"] == "dev":
0048             source_connect = "frontier://FrontierPrep/CMS_CONDITIONS"
0049         else:
0050             source_connect = tag["connect"]
0051 
0052         cmd = ("conddb_import",
0053                "-f", source_connect,
0054                "-c", result[record]["connect"],
0055                "-i", tag["tag"],
0056                "-t", result[record]["tag"],
0057                "-b", str(run_number),
0058                "-e", str(run_number))
0059         run_checked(cmd)
0060     if len(inputs) > 0:
0061         run_checked(["sqlite3", output_db, "update iov set since=1"])
0062 
0063     return result
0064 
0065 
0066 def run_checked(cmd, suppress_stderr = False):
0067     """Run `cmd` and exit in case of failures.
0068 
0069     Arguments:
0070     - `cmd`: list containing the strings of the command
0071     - `suppress_stderr`: suppress output from stderr
0072     """
0073 
0074     try:
0075         with open(os.devnull, "w") as devnull:
0076             if suppress_stderr:
0077                 subprocess.check_call(cmd, stdout = devnull, stderr = devnull)
0078             else:
0079                 subprocess.check_call(cmd, stdout = devnull)
0080     except subprocess.CalledProcessError as e:
0081         print("Problem in running the following command:")
0082         print(" ".join(e.cmd))
0083         sys.exit(1)
0084 
0085 
0086 def get_process_object(cfg):
0087     """Returns cms.Process object defined in `cfg`.
0088 
0089     Arguments:
0090     - `cfg`: path to CMSSW config file
0091     """
0092 
0093     sys.path.append(os.path.dirname(cfg)) # add location to python path
0094     cache_stdout = sys.stdout
0095     sys.stdout = open(os.devnull, "w")    # suppress unwanted output
0096     try:
0097         __configuration = \
0098             importlib.import_module(os.path.splitext(os.path.basename(cfg))[0])
0099     except Exception as e:
0100         print("Problem detected in configuration file '{0}'.".format(cfg))
0101         raise e
0102     sys.stdout = cache_stdout
0103     sys.path.pop()                        # clean up python path again
0104     try:
0105         os.remove(cfg+"c")                # try to remove temporary .pyc file
0106     except OSError as e:
0107         if e.args == (2, "No such file or directory"): pass
0108         else: raise
0109 
0110     return __configuration.process
0111 
0112 
0113 def make_unique_runranges(ali_producer):
0114     """Derive unique run ranges from AlignmentProducer PSet.
0115 
0116     Arguments:
0117     - `ali_producer`: cms.PSet containing AlignmentProducer configuration
0118     """
0119 
0120     if (hasattr(ali_producer, "RunRangeSelection") and
0121         len(ali_producer.RunRangeSelection) > 0):
0122         iovs = set([int(iov)
0123                     for sel in ali_producer.RunRangeSelection
0124                     for iov in sel.RunRanges])
0125         if len(iovs) == 0: return [1] # single IOV starting from run 1
0126         return sorted(iovs)
0127     else:
0128         return [1]                    # single IOV starting from run 1
0129 
0130 
0131 def get_tags(global_tag, records):
0132     """Get tags for `records` contained in `global_tag`.
0133 
0134     Arguments:
0135     - `global_tag`: global tag of interest
0136     - `records`: database records of interest
0137     """
0138 
0139     if len(records) == 0: return {} # avoid useless DB query
0140 
0141     # check for auto GT
0142     if global_tag.startswith("auto:"):
0143         import Configuration.AlCa.autoCond as AC
0144         try:
0145             global_tag = AC.autoCond[global_tag.split("auto:")[-1]]
0146         except KeyError:
0147             print("Unsupported auto GT:", global_tag)
0148             sys.exit(1)
0149 
0150     # setting up the DB session
0151     con = conddb.connect(url = conddb.make_url())
0152     session = con.session()
0153     GlobalTagMap = session.get_dbtype(conddb.GlobalTagMap)
0154 
0155     # query tag names for records of interest contained in `global_tag`
0156     tags = session.query(GlobalTagMap.record, GlobalTagMap.tag_name).\
0157            filter(GlobalTagMap.global_tag_name == global_tag,
0158                   GlobalTagMap.record.in_(records)).all()
0159 
0160     # closing the DB session
0161     session.close()
0162 
0163     return {item[0]: {"tag": item[1], "connect": "pro"} for item in tags}
0164 
0165 
0166 def get_iovs(db, tag):
0167     """Retrieve the list of IOVs from `db` for `tag`.
0168 
0169     Arguments:
0170     - `db`: database connection string
0171     - `tag`: tag of database record
0172     """
0173 
0174     db = db.replace("sqlite_file:", "").replace("sqlite:", "")
0175     db = db.replace("frontier://FrontierProd/CMS_CONDITIONS", "pro")
0176     db = db.replace("frontier://FrontierPrep/CMS_CONDITIONS", "dev")
0177 
0178     con = conddb.connect(url = conddb.make_url(db))
0179     session = con.session()
0180     IOV = session.get_dbtype(conddb.IOV)
0181 
0182     iovs = set(session.query(IOV.since).filter(IOV.tag_name == tag).all())
0183     if len(iovs) == 0:
0184         print("No IOVs found for tag '"+tag+"' in database '"+db+"'.")
0185         sys.exit(1)
0186 
0187     session.close()
0188 
0189     return sorted([int(item[0]) for item in iovs])
0190 
0191 
0192 def replace_factors(product_string, name, value):
0193     """Takes a `product_string` and replaces all factors with `name` by `value`.
0194 
0195     Arguments:
0196     - `product_string`: input string containing a product
0197     - `name`: name of the factor
0198     - `value`: value of the factor
0199     """
0200 
0201     value = str(value)                                 # ensure it's a string
0202     return re.sub(r"^"+name+r"$", value,               # single factor
0203                   re.sub(r"[*]"+name+r"$", r"*"+value, # rhs
0204                          re.sub(r"^"+name+r"[*]", value+r"*", # lhs
0205                                 re.sub(r"[*]"+name+r"[*]", r"*"+value+r"*",
0206                                        product_string))))
0207 
0208 def compute_product_string(product_string):
0209     """Takes `product_string` and returns the product of the factors as string.
0210 
0211     Arguments:
0212     - `product_string`: string containing product ('<factor>*<factor>*...')
0213     """
0214 
0215     factors = [float(f) for f in product_string.split("*")]
0216     return str(reduce(lambda x,y: x*y, factors))
0217 
0218 
0219 def check_proxy():
0220     """Check if GRID proxy has been initialized."""
0221 
0222     try:
0223         with open(os.devnull, "w") as dump:
0224             subprocess.check_call(["voms-proxy-info", "--exists"],
0225                                   stdout = dump, stderr = dump)
0226     except subprocess.CalledProcessError:
0227         return False
0228     return True
0229 
0230 
0231 def remove_existing_object(path):
0232     """
0233     Tries to remove file or directory located at `path`. If the user
0234     has no delete permissions, the object is moved to a backup
0235     file. If this fails it tries 5 times in total and then asks to
0236     perform a cleanup by a user with delete permissions.
0237 
0238     Arguments:
0239     - `name`: name of the object to be (re)moved
0240     """
0241 
0242     if os.path.exists(path):
0243         remove_method = shutil.rmtree if os.path.isdir(path) else os.remove
0244         move_method = shutil.move if os.path.isdir(path) else os.rename
0245         try:
0246             remove_method(path)
0247         except OSError as e:
0248             if e.args != (13, "Permission denied"): raise
0249             backup_path = path.rstrip("/")+"~"
0250             for _ in range(5):
0251                 try:
0252                     if os.path.exists(backup_path): remove_method(backup_path)
0253                     move_method(path, backup_path)
0254                     break
0255                 except OSError as e:
0256                     if e.args != (13, "Permission denied"): raise
0257                     backup_path += "~"
0258         if os.path.exists(path):
0259             msg = ("Cannot remove '{}' due to missing 'delete' ".format(path)
0260                    +"permissions and the limit of 5 backups is reached. Please "
0261                    "ask a user with 'delete' permissions to clean up.")
0262             print(msg)
0263             sys.exit(1)