File indexing completed on 2024-11-25 02:29:03
0001 from builtins import range
0002 import os
0003 import re
0004 import sys
0005 import shutil
0006 import importlib
0007 import sqlalchemy
0008 import subprocess
0009 import CondCore.Utilities.conddblib as conddb
0010 from functools import reduce
0011
0012
0013 def create_single_iov_db(inputs, run_number, output_db):
0014 """Create an sqlite file with single-IOV tags for alignment payloads.
0015
0016 Arguments:
0017 - `inputs`: dictionary with input needed for payload extraction
0018 - `run_number`: run for which the IOVs are selected
0019 - `output_db`: name of the output sqlite file
0020 """
0021
0022
0023 for record,tag in inputs.items():
0024 run_is_covered = False
0025 for iov in reversed(tag["iovs"]):
0026 if iov <= run_number:
0027 tag["since"] = str(iov)
0028 run_is_covered = True
0029 break
0030 if not run_is_covered:
0031 msg = ("Run number {0:d} is not covered in '{1:s}' ({2:s}) from"
0032 " '{3:s}'.".format(run_number, tag["tag"], record,
0033 global_tag))
0034 print(msg)
0035 print("Aborting...")
0036 sys.exit(1)
0037
0038 result = {}
0039 remove_existing_object(output_db)
0040
0041 for record,tag in inputs.items():
0042 result[record] = {"connect": "sqlite_file:"+output_db,
0043 "tag": "_".join([tag["tag"], tag["since"]])}
0044
0045 if tag["connect"] == "pro":
0046 source_connect = "frontier://FrontierProd/CMS_CONDITIONS"
0047 elif tag["connect"] == "dev":
0048 source_connect = "frontier://FrontierPrep/CMS_CONDITIONS"
0049 else:
0050 source_connect = tag["connect"]
0051
0052 cmd = ("conddb_import",
0053 "-f", source_connect,
0054 "-c", result[record]["connect"],
0055 "-i", tag["tag"],
0056 "-t", result[record]["tag"],
0057 "-b", str(run_number),
0058 "-e", str(run_number))
0059 run_checked(cmd)
0060 if len(inputs) > 0:
0061 run_checked(["sqlite3", output_db, "update iov set since=1"])
0062
0063 return result
0064
0065
0066 def run_checked(cmd, suppress_stderr = False):
0067 """Run `cmd` and exit in case of failures.
0068
0069 Arguments:
0070 - `cmd`: list containing the strings of the command
0071 - `suppress_stderr`: suppress output from stderr
0072 """
0073
0074 try:
0075 with open(os.devnull, "w") as devnull:
0076 if suppress_stderr:
0077 subprocess.check_call(cmd, stdout = devnull, stderr = devnull)
0078 else:
0079 subprocess.check_call(cmd, stdout = devnull)
0080 except subprocess.CalledProcessError as e:
0081 print("Problem in running the following command:")
0082 print(" ".join(e.cmd))
0083 sys.exit(1)
0084
0085
0086 def get_process_object(cfg):
0087 """Returns cms.Process object defined in `cfg`.
0088
0089 Arguments:
0090 - `cfg`: path to CMSSW config file
0091 """
0092
0093 sys.path.append(os.path.dirname(cfg))
0094 cache_stdout = sys.stdout
0095 sys.stdout = open(os.devnull, "w")
0096 try:
0097 __configuration = \
0098 importlib.import_module(os.path.splitext(os.path.basename(cfg))[0])
0099 except Exception as e:
0100 print("Problem detected in configuration file '{0}'.".format(cfg))
0101 raise e
0102 sys.stdout = cache_stdout
0103 sys.path.pop()
0104 try:
0105 os.remove(cfg+"c")
0106 except OSError as e:
0107 if e.args == (2, "No such file or directory"): pass
0108 else: raise
0109
0110 return __configuration.process
0111
0112
0113 def make_unique_runranges(ali_producer):
0114 """Derive unique run ranges from AlignmentProducer PSet.
0115
0116 Arguments:
0117 - `ali_producer`: cms.PSet containing AlignmentProducer configuration
0118 """
0119
0120 if (hasattr(ali_producer, "RunRangeSelection") and
0121 len(ali_producer.RunRangeSelection) > 0):
0122 iovs = set([int(iov)
0123 for sel in ali_producer.RunRangeSelection
0124 for iov in sel.RunRanges])
0125 if len(iovs) == 0: return [1]
0126 return sorted(iovs)
0127 else:
0128 return [1]
0129
0130
0131 def get_tags(global_tag, records):
0132 """Get tags for `records` contained in `global_tag`.
0133
0134 Arguments:
0135 - `global_tag`: global tag of interest
0136 - `records`: database records of interest
0137 """
0138
0139 if len(records) == 0: return {}
0140
0141
0142 if global_tag.startswith("auto:"):
0143 import Configuration.AlCa.autoCond as AC
0144 try:
0145 global_tag = AC.autoCond[global_tag.split("auto:")[-1]]
0146 except KeyError:
0147 print("Unsupported auto GT:", global_tag)
0148 sys.exit(1)
0149
0150
0151 con = conddb.connect(url = conddb.make_url())
0152 session = con.session()
0153 GlobalTagMap = session.get_dbtype(conddb.GlobalTagMap)
0154
0155
0156 tags = session.query(GlobalTagMap.record, GlobalTagMap.tag_name).\
0157 filter(GlobalTagMap.global_tag_name == global_tag,
0158 GlobalTagMap.record.in_(records)).all()
0159
0160
0161 session.close()
0162
0163 return {item[0]: {"tag": item[1], "connect": "pro"} for item in tags}
0164
0165
0166 def get_iovs(db, tag):
0167 """Retrieve the list of IOVs from `db` for `tag`.
0168
0169 Arguments:
0170 - `db`: database connection string
0171 - `tag`: tag of database record
0172 """
0173
0174 db = db.replace("sqlite_file:", "").replace("sqlite:", "")
0175 db = db.replace("frontier://FrontierProd/CMS_CONDITIONS", "pro")
0176 db = db.replace("frontier://FrontierPrep/CMS_CONDITIONS", "dev")
0177
0178 con = conddb.connect(url = conddb.make_url(db))
0179 session = con.session()
0180 IOV = session.get_dbtype(conddb.IOV)
0181
0182 iovs = set(session.query(IOV.since).filter(IOV.tag_name == tag).all())
0183 if len(iovs) == 0:
0184 print("No IOVs found for tag '"+tag+"' in database '"+db+"'.")
0185 sys.exit(1)
0186
0187 session.close()
0188
0189 return sorted([int(item[0]) for item in iovs])
0190
0191
0192 def replace_factors(product_string, name, value):
0193 """Takes a `product_string` and replaces all factors with `name` by `value`.
0194
0195 Arguments:
0196 - `product_string`: input string containing a product
0197 - `name`: name of the factor
0198 - `value`: value of the factor
0199 """
0200
0201 value = str(value)
0202 return re.sub(r"^"+name+r"$", value,
0203 re.sub(r"[*]"+name+r"$", r"*"+value,
0204 re.sub(r"^"+name+r"[*]", value+r"*",
0205 re.sub(r"[*]"+name+r"[*]", r"*"+value+r"*",
0206 product_string))))
0207
0208 def compute_product_string(product_string):
0209 """Takes `product_string` and returns the product of the factors as string.
0210
0211 Arguments:
0212 - `product_string`: string containing product ('<factor>*<factor>*...')
0213 """
0214
0215 factors = [float(f) for f in product_string.split("*")]
0216 return str(reduce(lambda x,y: x*y, factors))
0217
0218
0219 def check_proxy():
0220 """Check if GRID proxy has been initialized."""
0221
0222 try:
0223 with open(os.devnull, "w") as dump:
0224 subprocess.check_call(["voms-proxy-info", "--exists"],
0225 stdout = dump, stderr = dump)
0226 except subprocess.CalledProcessError:
0227 return False
0228 return True
0229
0230
0231 def remove_existing_object(path):
0232 """
0233 Tries to remove file or directory located at `path`. If the user
0234 has no delete permissions, the object is moved to a backup
0235 file. If this fails it tries 5 times in total and then asks to
0236 perform a cleanup by a user with delete permissions.
0237
0238 Arguments:
0239 - `name`: name of the object to be (re)moved
0240 """
0241
0242 if os.path.exists(path):
0243 remove_method = shutil.rmtree if os.path.isdir(path) else os.remove
0244 move_method = shutil.move if os.path.isdir(path) else os.rename
0245 try:
0246 remove_method(path)
0247 except OSError as e:
0248 if e.args != (13, "Permission denied"): raise
0249 backup_path = path.rstrip("/")+"~"
0250 for _ in range(5):
0251 try:
0252 if os.path.exists(backup_path): remove_method(backup_path)
0253 move_method(path, backup_path)
0254 break
0255 except OSError as e:
0256 if e.args != (13, "Permission denied"): raise
0257 backup_path += "~"
0258 if os.path.exists(path):
0259 msg = ("Cannot remove '{}' due to missing 'delete' ".format(path)
0260 +"permissions and the limit of 5 backups is reached. Please "
0261 "ask a user with 'delete' permissions to clean up.")
0262 print(msg)
0263 sys.exit(1)