|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
import FWCore.ParameterSet.Config as cms
from Configuration.StandardSequences.FrontierConditions_GlobalTag_cff import *
from Configuration.AlCa.autoCond import autoCond
# Default Express GT: it is the GT that will be used in case we are not able
# to retrieve the one used at Tier0.
# It should be kept in synch with Express processing at Tier0: what the url
# https://cmsweb.cern.ch/t0wmadatasvc/prod/express_config
# would tell you.
GlobalTag.globaltag = autoCond['run3_data_express']
# ===== auto -> Automatically get the GT string from current Tier0 configuration via a Tier0Das call.
# This needs a valid proxy to access the cern.ch network from the .cms one.
#
auto=False
# The implementation of the class is reused from the condition upload service.
#TODO: make this class a common utility under Conditions or Config.DP
import json
import os
import pycurl
import subprocess
import sys
import time
tier0Url = 'https://cmsweb.cern.ch/t0wmadatasvc/prod/'
class Tier0Error(Exception):
'''Tier0 exception.
'''
def __init__(self, message):
self.args = (message, )
def unique(seq, keepstr=True):
t = type(seq)
if t in (unicode, str):
t = (list, t('').join)[bool(keepstr)]
try:
remaining = set(seq)
seen = set()
return t(c for c in seq if (c in remaining and not remaining.remove(c)))
except TypeError: # hashing didn't work, see if seq is sortable
try:
from itertools import groupby
s = sorted(enumerate(seq),key=lambda i_v1:(i_v1[1],i_v1[0]))
return t(next(g) for k,g in groupby(s, lambda i_v: i_v[1]))
except: # not sortable, use brute force
seen = []
return t(c for c in seq if not (c in seen or seen.append(c)))
class Tier0Handler( object ):
def __init__( self, uri, timeOut, retries, retryPeriod, proxy, debug ):
"""
Parameters:
uri: Tier0DataSvc URI;
timeOut: time out for Tier0DataSvc HTTPS calls;
retries: maximum retries for Tier0DataSvc HTTPS calls;
retryPeriod: sleep time between two Tier0DataSvc HTTPS calls;
proxy: HTTP proxy for accessing Tier0DataSvc HTTPS calls;
debug: if set to True, enables debug information.
"""
self._uri = uri
self._timeOut = timeOut
self._retries = retries
self._retryPeriod = retryPeriod
self._proxy = proxy
self._debug = debug
def setDebug( self ):
self._debug = True
def unsetDebug( self ):
self._debug = False
def setProxy( self, proxy ):
self._proxy = proxy
def _queryTier0DataSvc( self, url ):
"""
Queries Tier0DataSvc.
url: Tier0DataSvc URL.
@returns: dictionary, from whence the required information must be retrieved according to the API call.
Raises if connection error, bad response, or timeout after retries occur.
"""
userAgent = "User-Agent: DQMIntegration/2.0 python/%d.%d.%d PycURL/%s" % ( sys.version_info[ :3 ] + ( pycurl.version_info()[ 1 ], ) )
proxy = ""
if self._proxy: proxy = ' --proxy %s ' % self._proxy
debug = " -s -S "
if self._debug: debug = " -v "
cmd = '/usr/bin/curl -k -L --user-agent "%s" %s --connect-timeout %i --retry %i %s %s ' % (userAgent, proxy, self._timeOut, self._retries, debug, url)
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(stdoutdata, stderrdata) = process.communicate()
retcode = process.returncode
if retcode != 0 or stderrdata:
msg = "looks like curl returned an error: retcode=%s" % (retcode,)
msg += ' msg = "'+str(stderrdata)+'"'
raise Tier0Error(msg)
return json.loads( ''.join(stdoutdata).replace( "'", '"').replace(' None', ' "None"') )
def getFirstSafeRun( self ):
"""
Queries Tier0DataSvc to get the first condition safe run.
Parameters:
@returns: integer, the run number.
Raises if connection error, bad response, timeout after retries occur, or if the run number is not available.
"""
firstConditionSafeRunAPI = "firstconditionsaferun"
safeRunDict = self._queryTier0DataSvc( os.path.join( self._uri, firstConditionSafeRunAPI ) )
if safeRunDict is None:
errStr = """First condition safe run is not available in Tier0DataSvc from URL \"%s\" """ %( os.path.join( self._uri, firstConditionSafeRunAPI ), )
if self._proxy:
errStr += """ using proxy \"%s\".""" %( str( self._proxy ), )
raise Tier0Error( errStr )
return int(safeRunDict['result'][0])
def getGlobalTag( self, config ):
"""
Queries Tier0DataSvc to get the most recent Global Tag for a given workflow.
Parameters:
config: Tier0DataSvc API call for the workflow to be looked for;
@returns: a string with the Global Tag name.
Raises if connection error, bad response, timeout after retries occur, or if no Global Tags are available.
"""
data = self._queryTier0DataSvc( os.path.join( self._uri, config ) )
gtnames = sorted(unique( [ str( di[ 'global_tag' ] ) for di in data['result'] if di[ 'global_tag' ] is not None ] ))
try:
recentGT = gtnames[-1]
return recentGT
except IndexError:
errStr = """No Global Tags for \"%s\" are available in Tier0DataSvc from URL \"%s\" """ %( config, os.path.join( self._uri, config ) )
if self._proxy:
errStr += """ using proxy \"%s\".""" %( str( self._proxy ), )
raise Tier0Error( errStr )
if auto:
proxyurl = None
if 'http_proxy' in os.environ:
proxyurl = os.environ[ 'http_proxy' ]
t0 = Tier0Handler( tier0Url, 5, 5, 5, proxyurl, False )
try:
# Get the express GT from Tie0 DataService API
GlobalTag.globaltag = cms.string( t0.getGlobalTag( 'express_config' ) )
print("The query to the Tier0 DataService returns the express GT: \"%s\"" % ( GlobalTag.globaltag.value(), ))
except Tier0Error as error:
# the web query did not succeed, fall back to the default
print("Error in querying the Tier0 DataService")
print(error)
print("Falling back to the default value of the express GT: \"%s\"" % ( GlobalTag.globaltag.value(), ))
else:
print("Using hardcoded GT: \"%s\"" % GlobalTag.globaltag.value())
|