python/config/FrontierCondition_GT_autoExpress_cfi.py

0001 from __future__ import print_function
0002 import FWCore.ParameterSet.Config as cms
0003 from Configuration.StandardSequences.FrontierConditions_GlobalTag_cff import *
0004 from Configuration.AlCa.autoCond import autoCond
0005
0006 # Default Express GT: it is the GT that will be used in case we are not able
0007 # to retrieve the one used at Tier0.
0008 # It should be kept in synch with Express processing at Tier0: what the url
0009 # https://cmsweb.cern.ch/t0wmadatasvc/prod/express_config
0010 # would tell you.
0011 GlobalTag.globaltag = autoCond['run3_data_express']
0012
0013 # ===== auto -> Automatically get the GT string from current Tier0 configuration via a Tier0Das call.
0014 #       This needs a valid proxy to access the cern.ch network from the .cms one.
0015 #
0016 auto=False
0017
0018 # The implementation of the class is reused from the condition upload service.
0019 #TODO: make this class a common utility under Conditions or Config.DP
0020 import json
0021 import os
0022 import pycurl
0023 import subprocess
0024 import sys
0025 import time
0026
0027 tier0Url = 'https://cmsweb.cern.ch/t0wmadatasvc/prod/'
0028
0029 class Tier0Error(Exception):
0030     '''Tier0 exception.
0031     '''
0032
0033     def __init__(self, message):
0034         self.args = (message, )
0035
0036 def unique(seq, keepstr=True):
0037     t = type(seq)
0038     if t in (unicode, str):
0039         t = (list, t('').join)[bool(keepstr)]
0040     try:
0041         remaining = set(seq)
0042         seen = set()
0043         return t(c for c in seq if (c in remaining and not remaining.remove(c)))
0044     except TypeError: # hashing didn't work, see if seq is sortable
0045         try:
0046             from itertools import groupby
0047             s = sorted(enumerate(seq),key=lambda i_v1:(i_v1[1],i_v1[0]))
0048             return t(next(g) for k,g in groupby(s, lambda i_v: i_v[1]))
0049         except:  # not sortable, use brute force
0050             seen = []
0051             return t(c for c in seq if not (c in seen or seen.append(c)))
0052
0053 class Tier0Handler( object ):
0054
0055     def __init__( self, uri, timeOut, retries, retryPeriod, proxy, debug ):
0056         """
0057         Parameters:
0058         uri: Tier0DataSvc URI;
0059         timeOut: time out for Tier0DataSvc HTTPS calls;
0060         retries: maximum retries for Tier0DataSvc HTTPS calls;
0061         retryPeriod: sleep time between two Tier0DataSvc HTTPS calls;
0062         proxy: HTTP proxy for accessing Tier0DataSvc HTTPS calls;
0063         debug: if set to True, enables debug information.
0064         """
0065         self._uri = uri
0066         self._timeOut = timeOut
0067         self._retries = retries
0068         self._retryPeriod = retryPeriod
0069         self._proxy = proxy
0070         self._debug = debug
0071
0072     def setDebug( self ):
0073         self._debug = True
0074
0075     def unsetDebug( self ):
0076         self._debug = False
0077
0078     def setProxy( self, proxy ):
0079         self._proxy = proxy
0080
0081     def _queryTier0DataSvc( self, url ):
0082         """
0083         Queries Tier0DataSvc.
0084         url: Tier0DataSvc URL.
0085         @returns: dictionary, from whence the required information must be retrieved according to the API call.
0086         Raises if connection error, bad response, or timeout after retries occur.
0087         """
0088
0089         userAgent = "User-Agent: DQMIntegration/2.0 python/%d.%d.%d PycURL/%s" % ( sys.version_info[ :3 ] + ( pycurl.version_info()[ 1 ], ) )
0090
0091         proxy = ""
0092         if self._proxy: proxy = ' --proxy %s ' % self._proxy
0093
0094         debug = " -s -S "
0095         if self._debug: debug = " -v "
0096
0097         cmd = '/usr/bin/curl -k -L --user-agent "%s" %s --connect-timeout %i --retry %i %s %s ' % (userAgent, proxy, self._timeOut, self._retries, debug, url)
0098
0099         process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
0100         (stdoutdata, stderrdata) =  process.communicate()
0101         retcode = process.returncode
0102
0103         if retcode != 0 or stderrdata:
0104            msg = "looks like curl returned an error: retcode=%s" % (retcode,)
0105            msg += ' msg = "'+str(stderrdata)+'"'
0106            raise Tier0Error(msg)
0107
0108         return json.loads( ''.join(stdoutdata).replace( "'", '"').replace(' None', ' "None"') )
0109
0110     def getFirstSafeRun( self ):
0111         """
0112         Queries Tier0DataSvc to get the first condition safe run.
0113         Parameters:
0114         @returns: integer, the run number.
0115         Raises if connection error, bad response, timeout after retries occur, or if the run number is not available.
0116         """
0117         firstConditionSafeRunAPI = "firstconditionsaferun"
0118         safeRunDict = self._queryTier0DataSvc( os.path.join( self._uri, firstConditionSafeRunAPI ) )
0119         if safeRunDict is None:
0120             errStr = """First condition safe run is not available in Tier0DataSvc from URL \"%s\" """ %( os.path.join( self._uri, firstConditionSafeRunAPI ), )
0121             if self._proxy:
0122                 errStr += """ using proxy \"%s\".""" %( str( self._proxy ), )
0123             raise Tier0Error( errStr )
0124         return int(safeRunDict['result'][0])
0125
0126     def getGlobalTag( self, config ):
0127         """
0128         Queries Tier0DataSvc to get the most recent Global Tag for a given workflow.
0129         Parameters:
0130         config: Tier0DataSvc API call for the workflow to be looked for;
0131         @returns: a string with the Global Tag name.
0132         Raises if connection error, bad response, timeout after retries occur, or if no Global Tags are available.
0133         """
0134         data = self._queryTier0DataSvc( os.path.join( self._uri, config ) )
0135         gtnames = sorted(unique( [ str( di[ 'global_tag' ] ) for di in data['result'] if di[ 'global_tag' ] is not None ] ))
0136         try:
0137             recentGT = gtnames[-1]
0138             return recentGT
0139         except IndexError:
0140             errStr = """No Global Tags for \"%s\" are available in Tier0DataSvc from URL \"%s\" """ %( config, os.path.join( self._uri, config ) )
0141             if self._proxy:
0142                 errStr += """ using proxy \"%s\".""" %( str( self._proxy ), )
0143         raise Tier0Error( errStr )
0144
0145 if auto:
0146     proxyurl = None
0147     if 'http_proxy' in os.environ:
0148         proxyurl = os.environ[ 'http_proxy' ]
0149     t0 = Tier0Handler( tier0Url, 5, 5, 5, proxyurl, False )
0150
0151     try:
0152         # Get the express GT from Tie0 DataService API
0153         GlobalTag.globaltag = cms.string( t0.getGlobalTag( 'express_config' ) )
0154         print("The query to the Tier0 DataService returns the express GT: \"%s\"" % ( GlobalTag.globaltag.value(), ))
0155     except Tier0Error as error:
0156         # the web query did not succeed, fall back to the default
0157         print("Error in querying the Tier0 DataService")
0158         print(error)
0159         print("Falling back to the default value of the express GT: \"%s\"" % ( GlobalTag.globaltag.value(), ))
0160 else:
0161     print("Using hardcoded GT: \"%s\"" % GlobalTag.globaltag.value())