python/config/FrontierCondition_GT_autoExpress_cfi.py

0001 import FWCore.ParameterSet.Config as cms
0002 from Configuration.StandardSequences.FrontierConditions_GlobalTag_cff import *
0003 from Configuration.AlCa.autoCond import autoCond
0004
0005 # Default Express GT: it is the GT that will be used in case we are not able
0006 # to retrieve the one used at Tier0.
0007 # It should be kept in synch with Express processing at Tier0: what the url
0008 # https://cmsweb.cern.ch/t0wmadatasvc/prod/express_config
0009 # would tell you.
0010 GlobalTag.globaltag = autoCond['run3_data_express']
0011
0012 # ===== auto -> Automatically get the GT string from current Tier0 configuration via a Tier0Das call.
0013 #       This needs a valid proxy to access the cern.ch network from the .cms one.
0014 #
0015 auto=False
0016
0017 # The implementation of the class is reused from the condition upload service.
0018 #TODO: make this class a common utility under Conditions or Config.DP
0019 import json
0020 import os
0021 import pycurl
0022 import subprocess
0023 import sys
0024 import time
0025
0026 tier0Url = 'https://cmsweb.cern.ch/t0wmadatasvc/prod/'
0027
0028 class Tier0Error(Exception):
0029     '''Tier0 exception.
0030     '''
0031
0032     def __init__(self, message):
0033         self.args = (message, )
0034
0035 def unique(seq, keepstr=True):
0036     t = type(seq)
0037     if t in (unicode, str):
0038         t = (list, t('').join)[bool(keepstr)]
0039     try:
0040         remaining = set(seq)
0041         seen = set()
0042         return t(c for c in seq if (c in remaining and not remaining.remove(c)))
0043     except TypeError: # hashing didn't work, see if seq is sortable
0044         try:
0045             from itertools import groupby
0046             s = sorted(enumerate(seq),key=lambda i_v1:(i_v1[1],i_v1[0]))
0047             return t(next(g) for k,g in groupby(s, lambda i_v: i_v[1]))
0048         except:  # not sortable, use brute force
0049             seen = []
0050             return t(c for c in seq if not (c in seen or seen.append(c)))
0051
0052 class Tier0Handler( object ):
0053
0054     def __init__( self, uri, timeOut, retries, retryPeriod, proxy, debug ):
0055         """
0056         Parameters:
0057         uri: Tier0DataSvc URI;
0058         timeOut: time out for Tier0DataSvc HTTPS calls;
0059         retries: maximum retries for Tier0DataSvc HTTPS calls;
0060         retryPeriod: sleep time between two Tier0DataSvc HTTPS calls;
0061         proxy: HTTP proxy for accessing Tier0DataSvc HTTPS calls;
0062         debug: if set to True, enables debug information.
0063         """
0064         self._uri = uri
0065         self._timeOut = timeOut
0066         self._retries = retries
0067         self._retryPeriod = retryPeriod
0068         self._proxy = proxy
0069         self._debug = debug
0070
0071     def setDebug( self ):
0072         self._debug = True
0073
0074     def unsetDebug( self ):
0075         self._debug = False
0076
0077     def setProxy( self, proxy ):
0078         self._proxy = proxy
0079
0080     def _queryTier0DataSvc( self, url ):
0081         """
0082         Queries Tier0DataSvc.
0083         url: Tier0DataSvc URL.
0084         @returns: dictionary, from whence the required information must be retrieved according to the API call.
0085         Raises if connection error, bad response, or timeout after retries occur.
0086         """
0087
0088         userAgent = "User-Agent: DQMIntegration/2.0 python/%d.%d.%d PycURL/%s" % ( sys.version_info[ :3 ] + ( pycurl.version_info()[ 1 ], ) )
0089
0090         proxy = ""
0091         if self._proxy: proxy = ' --proxy %s ' % self._proxy
0092
0093         debug = " -s -S "
0094         if self._debug: debug = " -v "
0095
0096         cmd = '/usr/bin/curl -k -L --user-agent "%s" %s --connect-timeout %i --retry %i %s %s ' % (userAgent, proxy, self._timeOut, self._retries, debug, url)
0097
0098         process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
0099         (stdoutdata, stderrdata) =  process.communicate()
0100         retcode = process.returncode
0101
0102         if retcode != 0 or stderrdata:
0103            msg = "looks like curl returned an error: retcode=%s" % (retcode,)
0104            msg += ' msg = "'+str(stderrdata)+'"'
0105            raise Tier0Error(msg)
0106
0107         return json.loads( ''.join(stdoutdata).replace( "'", '"').replace(' None', ' "None"') )
0108
0109     def getFirstSafeRun( self ):
0110         """
0111         Queries Tier0DataSvc to get the first condition safe run.
0112         Parameters:
0113         @returns: integer, the run number.
0114         Raises if connection error, bad response, timeout after retries occur, or if the run number is not available.
0115         """
0116         firstConditionSafeRunAPI = "firstconditionsaferun"
0117         safeRunDict = self._queryTier0DataSvc( os.path.join( self._uri, firstConditionSafeRunAPI ) )
0118         if safeRunDict is None:
0119             errStr = """First condition safe run is not available in Tier0DataSvc from URL \"%s\" """ %( os.path.join( self._uri, firstConditionSafeRunAPI ), )
0120             if self._proxy:
0121                 errStr += """ using proxy \"%s\".""" %( str( self._proxy ), )
0122             raise Tier0Error( errStr )
0123         return int(safeRunDict['result'][0])
0124
0125     def getGlobalTag( self, config ):
0126         """
0127         Queries Tier0DataSvc to get the most recent Global Tag for a given workflow.
0128         Parameters:
0129         config: Tier0DataSvc API call for the workflow to be looked for;
0130         @returns: a string with the Global Tag name.
0131         Raises if connection error, bad response, timeout after retries occur, or if no Global Tags are available.
0132         """
0133         data = self._queryTier0DataSvc( os.path.join( self._uri, config ) )
0134         gtnames = sorted(unique( [ str( di[ 'global_tag' ] ) for di in data['result'] if di[ 'global_tag' ] is not None ] ))
0135         try:
0136             recentGT = gtnames[-1]
0137             return recentGT
0138         except IndexError:
0139             errStr = """No Global Tags for \"%s\" are available in Tier0DataSvc from URL \"%s\" """ %( config, os.path.join( self._uri, config ) )
0140             if self._proxy:
0141                 errStr += """ using proxy \"%s\".""" %( str( self._proxy ), )
0142         raise Tier0Error( errStr )
0143
0144 if auto:
0145     proxyurl = None
0146     if 'http_proxy' in os.environ:
0147         proxyurl = os.environ[ 'http_proxy' ]
0148     t0 = Tier0Handler( tier0Url, 5, 5, 5, proxyurl, False )
0149
0150     try:
0151         # Get the express GT from Tie0 DataService API
0152         GlobalTag.globaltag = cms.string( t0.getGlobalTag( 'express_config' ) )
0153         print("The query to the Tier0 DataService returns the express GT: \"%s\"" % ( GlobalTag.globaltag.value(), ))
0154     except Tier0Error as error:
0155         # the web query did not succeed, fall back to the default
0156         print("Error in querying the Tier0 DataService")
0157         print(error)
0158         print("Falling back to the default value of the express GT: \"%s\"" % ( GlobalTag.globaltag.value(), ))
0159 else:
0160     print("Using hardcoded GT: \"%s\"" % GlobalTag.globaltag.value())