Warning, /CondCore/SiStripPlugins/scripts/SiStripG2GainsValidator is written in an unsupported language. File is not indexed.
0001 #!/usr/bin/env python3
0002 from datetime import datetime
0003 import configparser as ConfigParser
0004 import json
0005 import optparse
0006 import os
0007 import sqlalchemy
0008 import string
0009 import subprocess
0010 import sys
0011 import CondCore.Utilities.conddblib as conddb
0012
0013 ##############################################
0014 def getCommandOutput(command):
0015 ##############################################
0016 """This function executes `command` and returns it output.
0017 Arguments:
0018 - `command`: Shell command to be invoked by this function.
0019 """
0020 child = os.popen(command)
0021 data = child.read()
0022 err = child.close()
0023 if err:
0024 print ('%s failed w/ exit code %d' % (command, err))
0025 sys.exit(1) # This will stop the script immediately with the failure exit code
0026 print(data)
0027 return data
0028
0029 ##############################################
0030 def getCerts() -> str:
0031 ##############################################
0032 cert_path = os.getenv('X509_USER_CERT', '')
0033 key_path = os.getenv('X509_USER_KEY', '')
0034
0035 certs = ""
0036 if cert_path:
0037 certs += f' --cert {cert_path}'
0038 else:
0039 print("No certificate, nor proxy provided for Tier0 access")
0040 if key_path:
0041 certs += f' --key {key_path}'
0042 return certs
0043
0044 ##############################################
0045 def build_curl_command(url, proxy="", certs="", timeout=30, retries=3, user_agent="MyUserAgent"):
0046 ##############################################
0047 """Builds the curl command with the appropriate proxy, certs, and options."""
0048 cmd = f'/usr/bin/curl -k -L --user-agent "{user_agent}" '
0049
0050 if proxy:
0051 cmd += f'--proxy {proxy} '
0052 else:
0053 cmd += f'{certs} '
0054
0055 cmd += f'--connect-timeout {timeout} --retry {retries} {url}'
0056 return cmd
0057
0058 ##############################################
0059 def get_hlt_fcsr(session):
0060 ##############################################
0061 RunInfo = session.get_dbtype(conddb.RunInfo)
0062 lastRun = session.query(sqlalchemy.func.max(RunInfo.run_number)).scalar()
0063 fcsr = lastRun+1
0064 return int(fcsr)
0065
0066 ##############################################
0067 def fetch_data_from_url(url, proxy="", certs=""):
0068 ##############################################
0069 cmd = build_curl_command(url, proxy=proxy, certs=certs)
0070
0071 try:
0072 out = subprocess.check_output(cmd, shell=True)
0073 except subprocess.CalledProcessError as e:
0074 print(f"Error executing curl command: {e}")
0075 return None
0076
0077 if not out.strip():
0078 print("Received an empty response from the server.")
0079 return None
0080
0081 try:
0082 response = json.loads(out)
0083 except json.JSONDecodeError as e:
0084 print(f"Failed to decode JSON: {e}")
0085 print(f"Raw output was: {out}")
0086 return None
0087
0088 # Ensure the expected structure is there
0089 if "result" not in response or len(response["result"]) == 0:
0090 print("Unexpected response format or empty result.")
0091 return None
0092
0093 return response["result"][0] # Return the first result entry (this is where individual functions can extract specific fields)
0094
0095 ##############################################
0096 def getFCSR(proxy="", certs=""):
0097 ##############################################
0098 url = "https://cmsweb.cern.ch/t0wmadatasvc/prod/firstconditionsaferun"
0099 response = fetch_data_from_url(url, proxy, certs)
0100
0101 if response is None:
0102 return None
0103
0104 return int(response) # Assuming 'response' is already an integer value in this case
0105
0106 ##############################################
0107 def getPromptGT(proxy="", certs=""):
0108 ##############################################
0109 url = "https://cmsweb.cern.ch/t0wmadatasvc/prod/reco_config"
0110 response = fetch_data_from_url(url, proxy, certs)
0111
0112 if response is None:
0113 return None
0114
0115 return response.get('global_tag')
0116
0117 ##############################################
0118 def getExpressGT(proxy="", certs=""):
0119 ##############################################
0120 url = "https://cmsweb.cern.ch/t0wmadatasvc/prod/express_config"
0121 response = fetch_data_from_url(url, proxy, certs)
0122
0123 if response is None:
0124 return None
0125
0126 return response.get('global_tag')
0127
0128 ##############################################
0129 def _getFCSR(proxy="", certs=""):
0130 ##############################################
0131 url = "https://cmsweb.cern.ch/t0wmadatasvc/prod/firstconditionsaferun"
0132 cmd = build_curl_command(url, proxy=proxy, certs=certs)
0133 out = subprocess.check_output(cmd, shell=True)
0134 response = json.loads(out)["result"][0]
0135 return int(response)
0136
0137 ##############################################
0138 def _getPromptGT(proxy="", certs=""):
0139 ##############################################
0140 url = "https://cmsweb.cern.ch/t0wmadatasvc/prod/reco_config"
0141 cmd = build_curl_command(url, proxy=proxy, certs=certs)
0142 out = subprocess.check_output(cmd, shell=True)
0143 response = json.loads(out)["result"][0]['global_tag']
0144 return response
0145
0146 ##############################################
0147 def _getExpressGT(proxy="", certs=""):
0148 ##############################################
0149 url = "https://cmsweb.cern.ch/t0wmadatasvc/prod/express_config"
0150 cmd = build_curl_command(url, proxy=proxy, certs=certs)
0151 out = subprocess.check_output(cmd, shell=True)
0152 response = json.loads(out)["result"][0]['global_tag']
0153 return response
0154
0155 ##############################################
0156 def resetSynchonization(db_name):
0157 ##############################################
0158 import sqlite3
0159
0160 # Connect to the SQLite database
0161 conn = sqlite3.connect(db_name)
0162
0163 # Create a cursor object to execute SQL commands
0164 cursor = conn.cursor()
0165
0166 # Execute the SQL command to update the database
0167 cursor.execute("UPDATE TAG SET SYNCHRONIZATION='any' WHERE SYNCHRONIZATION='express';")
0168
0169 # Commit the changes and close the connection
0170 conn.commit()
0171 conn.close()
0172
0173 ##############################################
0174 if __name__ == "__main__":
0175 ##############################################
0176
0177 parser = optparse.OptionParser(usage = 'Usage: %prog [options] <file> [<file> ...]\n')
0178
0179 parser.add_option('-t', '--validationTag',
0180 dest = 'validationTag',
0181 default = "SiStripApvGainAfterAbortGap_PCL_multirun_v0_prompt",
0182 help = 'validation tag',
0183 )
0184
0185 parser.add_option('-s', '--since',
0186 dest = 'since',
0187 default = -1,
0188 help = 'sinces to copy from validation tag',
0189 )
0190
0191 parser.add_option('-p', '--proxy',
0192 dest = 'proxy',
0193 default = "",
0194 help = 'proxy to use for curl requests',
0195 )
0196
0197 parser.add_option('-u', '--user-mode',
0198 dest='user_mode',
0199 action='store_true',
0200 default=False,
0201 help='Enable user mode with specific X509 user certificate and key')
0202
0203 (options, arguments) = parser.parse_args()
0204
0205 if options.user_mode:
0206 os.environ['X509_USER_KEY'] = os.path.expanduser('~/.globus/userkey.pem')
0207 os.environ['X509_USER_CERT'] = os.path.expanduser('~/.globus/usercert.pem')
0208 print("User mode enabled. Using X509_USER_KEY and X509_USER_CERT from ~/.globus/")
0209
0210 certs = ""
0211 if not options.proxy:
0212 certs = getCerts()
0213
0214 FCSR = getFCSR(proxy=options.proxy, certs=certs)
0215 promptGT = getPromptGT(proxy=options.proxy, certs=certs)
0216 expressGT = getExpressGT(proxy=options.proxy, certs=certs)
0217
0218 con = conddb.connect(url = conddb.make_url("pro"))
0219 session = con.session()
0220
0221 HLTFCSR = get_hlt_fcsr(session)
0222 print ("Current next HLT run", HLTFCSR, "| curret FCSR:", FCSR ,"| Express Global Tag",expressGT,"| Prompt Global Tag",promptGT)
0223 IOV = session.get_dbtype(conddb.IOV)
0224 TAG = session.get_dbtype(conddb.Tag)
0225 GT = session.get_dbtype(conddb.GlobalTag)
0226 GTMAP = session.get_dbtype(conddb.GlobalTagMap)
0227 RUNINFO = session.get_dbtype(conddb.RunInfo)
0228
0229 myGTMap = session.query(GTMAP.record, GTMAP.label, GTMAP.tag_name).\
0230 filter(GTMAP.global_tag_name == str(expressGT)).\
0231 order_by(GTMAP.record, GTMAP.label).\
0232 all()
0233
0234 # Check if the query result is empty
0235 if not myGTMap:
0236 print("No records found in the GTMap.")
0237 sys.exit(1)
0238
0239 ## connect to prep DB and get the list of IOVs to look at
0240 con2 = conddb.connect(url = conddb.make_url("dev"))
0241 session2 = con2.session()
0242 validationTagIOVs = session2.query(IOV.since,IOV.payload_hash,IOV.insertion_time).filter(IOV.tag_name == options.validationTag).all()
0243
0244 ### fill the list of IOVs to be validated
0245 IOVsToValidate=[]
0246 if(options.since==-1):
0247 IOVsToValidate.append(validationTagIOVs[-1][0])
0248 print("Changing the default validation tag since to:",IOVsToValidate[0])
0249
0250 else:
0251 for entry in validationTagIOVs:
0252 if(options.since!=1 and int(entry[0])>=int(options.since)):
0253 print("Appending to the validation list:",entry[0],entry[1],entry[2])
0254 IOVsToValidate.append(entry[0])
0255
0256 for element in myGTMap:
0257 #print element
0258 Record = element[0]
0259 Label = element[1]
0260 Tag = element[2]
0261 if(Record=="SiStripApvGain2Rcd"):
0262 TagIOVs = session.query(IOV.since,IOV.payload_hash,IOV.insertion_time).filter(IOV.tag_name == Tag).all()
0263 sorted_TagIOVs = sorted(TagIOVs, key=lambda x: x[0])
0264 lastG2Payload = sorted_TagIOVs[-1]
0265 print("Last G2 Prompt payload has IOV since:",lastG2Payload[0],"payload hash:",lastG2Payload[1],"insertion time:",lastG2Payload[2])
0266
0267 # Get and print the current working directory
0268 current_directory = os.getcwd()
0269 print("Current Working Directory:", current_directory)
0270
0271 # Construct the conddb_import command using f-strings for better readability
0272 command = (
0273 f'conddb_import -c sqlite_file:toCompare.db '
0274 f'-f frontier://FrontierProd/CMS_CONDITIONS '
0275 f'-i {Tag} -t {Tag} -b {lastG2Payload[0]}'
0276 )
0277 print(command)
0278 getCommandOutput(command)
0279
0280 # set syncrhonization to any
0281 resetSynchonization("toCompare.db")
0282
0283 for i,theValidationTagSince in enumerate(IOVsToValidate):
0284
0285 # Construct the conddb_import command, modifying the 'since' value if necessary
0286 since_value = HLTFCSR + i if theValidationTagSince < lastG2Payload[0] else theValidationTagSince
0287 command = (
0288 f'conddb_import -c sqlite_file:toCompare.db '
0289 f'-f frontier://FrontierPrep/CMS_CONDITIONS '
0290 f'-i {options.validationTag} -t {Tag} -b {since_value}'
0291 )
0292
0293 # Print and execute the conddb_import command
0294 if theValidationTagSince < lastG2Payload[0]:
0295 print("The last available IOV in the validation tag is older than the current last express IOV, taking Express FCSR (HLT) as a since!")
0296
0297 print(command)
0298 getCommandOutput(command)
0299
0300 # Construct the testCompareSiStripG2Gains.sh command, adjusting the 'since' value similarly
0301 since_value = HLTFCSR + i if theValidationTagSince < lastG2Payload[0] else theValidationTagSince
0302 command = (
0303 f'${{CMSSW_BASE}}/src/CondCore/SiStripPlugins/scripts/testCompareSiStripG2Gains.sh '
0304 f'{Tag} {lastG2Payload[0]} {since_value} {current_directory}/toCompare.db'
0305 )
0306
0307 # Print and execute the testCompareSiStripG2Gains.sh command
0308 print(command)
0309 getCommandOutput(command)
