File indexing completed on 2024-11-25 02:29:20
0001
0002
0003
0004 import shutil
0005 import glob
0006 import json
0007 import yaml
0008 import sys
0009 import sys
0010 import os
0011
0012 from importlib import import_module
0013 from argparse import ArgumentParser, RawTextHelpFormatter
0014
0015
0016 import pluginCondDBV2PyInterface
0017 pluginCondDBV2PyInterface.CMSSWInit()
0018
0019
0020
0021 def supress_output( f ):
0022 '''
0023 Temporarily disables stdout and stderr so that printouts from the plot
0024 plugin does not compromise the purity of our ssh stream if
0025 args.suppress_output is true
0026 '''
0027 def decorated( *fargs, **fkwargs ):
0028
0029 suppress = args.suppress_output
0030 if suppress:
0031
0032
0033 sys.stdout.flush()
0034
0035
0036 saved_stdout = os.dup( 1 )
0037 saved_stderr = os.dup( 2 )
0038
0039
0040 devnull = os.open( '/dev/null', os.O_WRONLY )
0041
0042
0043
0044 os.dup2( devnull, 1 )
0045 os.dup2( devnull, 2 )
0046
0047 result = f( *fargs, **fkwargs )
0048
0049 if suppress:
0050
0051
0052 os.close( devnull )
0053
0054
0055 os.dup2( saved_stdout, 1 )
0056 os.dup2( saved_stderr, 2 )
0057
0058 return result
0059
0060 return decorated
0061
0062
0063 @supress_output
0064 def deserialize_iovs(db, plugin_name, plot_name, tags, time_type, input_params):
0065 ''' Deserializes given iovs data and returns plot coordinates '''
0066
0067 output('Starting to deserialize iovs: ', '')
0068 output('db: ', db)
0069 output('plugin name: ', plugin_name)
0070 output('plot name: ', plot_name)
0071 output('tags: ', tags)
0072 output('tag time type: ', time_type)
0073
0074 plugin_base = import_module('pluginModule_PayloadInspector')
0075 output('PI plugin base: ', plugin_base)
0076
0077 plugin_obj = import_module(plugin_name)
0078 output('PI plugin object: ', plugin_obj)
0079
0080
0081 plot = getattr(plugin_obj, plot_name)()
0082 output('plot object: ', plot)
0083
0084 if db == "Prod":
0085 db_name = 'frontier://FrontierProd/CMS_CONDITIONS'
0086 elif db == 'Prep' :
0087 db_name = 'frontier://FrontierPrep/CMS_CONDITIONS'
0088 else:
0089 db_name = db
0090
0091 output('full DB name: ', db_name)
0092
0093 if input_params is not None:
0094 plot.setInputParamValues( input_params )
0095
0096 modv = getattr(plugin_base,'ModuleVersion')
0097
0098 success = False
0099 if modv.label == '1.0':
0100 if len(tags)==1:
0101 success = plot.process(db_name, tags[0][0], time_type, int(tags[0][1]), int(tags[0][2]) )
0102 elif len(tags)==2:
0103 success = plot.processTwoTags(db_name, tags[0][0], tags[1][0], int(tags[0][1]),int(tags[1][1]) )
0104 elif modv.label == '2.0':
0105 success = plot.process(db_name, tags)
0106
0107 output('plot processed data successfully: ', success)
0108 if not success:
0109 return False
0110
0111 result = plot.data()
0112 output('deserialized data: ', result)
0113 return result
0114
0115 def discover_plugins():
0116 ''' Returns a list of Payload Inspector plugin names
0117 Example:
0118 ['pluginBasicPayload_PayloadInspector', 'pluginBeamSpot_PayloadInspector', 'pluginSiStrip_PayloadInspector']
0119 '''
0120 architecture = os.environ.get('SCRAM_ARCH', None)
0121 output('architecture: ', architecture)
0122
0123 plugins = []
0124 releases = [
0125 os.environ.get('CMSSW_BASE', None),
0126 os.environ.get('CMSSW_RELEASE_BASE', None),
0127 os.environ.get('CMSSW_FULL_RELEASE_BASE', None)
0128 ]
0129
0130 for r in releases:
0131 if not r: continue
0132 output('* release: ', r)
0133
0134 path = os.path.join(r, 'lib', architecture)
0135 output('* full release path: ', path)
0136
0137 plugins += glob.glob(path + '/plugin*_PayloadInspector.so' )
0138 output('found plugins: ', plugins)
0139
0140
0141
0142 if(len(plugins)==0):
0143 output('# plugins found:',len(plugins))
0144 else:
0145 if r: break
0146
0147
0148
0149
0150 result = []
0151 for p in plugins:
0152 result.append(p.split('/')[-1].replace('.so', ''))
0153
0154 output('discovered plugins: ', result)
0155 return result
0156
0157 def discover():
0158 ''' Discovers object types and plots for a given cmssw release
0159 Example:
0160 {
0161 "BasicPayload": [
0162 {"plot": "plot_BeamSpot_x", "plot_type": "History",
0163 "single_iov": false, "plugin_name": "pluginBeamSpot_PayloadInspector",
0164 "title": "x vs run number"},
0165 ...
0166 ],
0167 ...
0168 }
0169 '''
0170 plugin_base = import_module('pluginModule_PayloadInspector')
0171 modv = getattr(plugin_base,'ModuleVersion')
0172 result = {}
0173 for plugin_name in discover_plugins():
0174 output(' - plugin name: ', plugin_name)
0175 plugin_obj = import_module(plugin_name)
0176 output('*** PI plugin object: ', plugin_obj)
0177 for plot in dir(plugin_obj):
0178 if 'plot_' not in plot: continue
0179 output(' - plot name: ', plot)
0180 plot_method= getattr(plugin_obj, plot)()
0181 output(' - plot object: ', plot_method)
0182 payload_type = plot_method.payloadType()
0183 output(' - payload type: ', payload_type)
0184 plot_title = plot_method.title()
0185 output(' - plot title: ', plot_title)
0186 plot_type = plot_method.type()
0187 output(' - plot type: ', plot_type)
0188 single_iov = plot_method.isSingleIov()
0189 output(' - is single iov: ', single_iov)
0190 two_tags = plot_method.isTwoTags()
0191 output(' - is Two Tags: ', two_tags)
0192 plot_dict = { 'plot': plot, 'plugin_name': plugin_name, 'title': plot_title, 'plot_type': plot_type, 'single_iov': single_iov, 'two_tags': two_tags }
0193 if modv.label == '2.0':
0194 input_params = plot_method.inputParams()
0195 output(' - input params: ', len(input_params))
0196 plot_dict[ 'input_params'] = input_params
0197 result.setdefault(payload_type, []).append( plot_dict )
0198 output('currently discovered info: ', result)
0199 output('*** final output:', '')
0200 return json.dumps(result)
0201
0202 def output(description, param):
0203 if args.verbose:
0204 print('')
0205 print(description, param)
0206
0207 if __name__ == '__main__':
0208
0209 description = '''
0210 Payload Inspector - data visualisation tool which is integrated into the cmsDbBrowser.
0211 It allows to display plots and monitor the calibration and alignment data.
0212
0213 You can access Payload Inspector with a link below:
0214 https://cms-conddb.cern.ch/cmsDbBrowser/payload_inspector/Prod
0215
0216 This script is a part of the Payload Inspector service and is responsible for:
0217 a) discovering PI objects that are available in a given cmssw release
0218 b) deserializing payload data which is later used as plot coordinates
0219 c) testing new PI objects which are under development
0220
0221 To test new PI objects please do the following:
0222 a) run ./getPayloadData.py --discover
0223 to check if your newly created object is found by the script.
0224 Please note that we strongly rely on naming conventions so if you don't
0225 see your object listed you probably misnamed it in objectType() method.
0226 Also all plot methods should start with "plot_" prefix.
0227
0228 b) second step is to test if it returns data correctly:
0229 run ./getPayloadData.py --plugin YourPIPluginName --plot YourObjectPlot --tag tagName --time_type Run --iovs '{"start_iov": "201", "end_iov": "801"}' --db Prod --test
0230
0231 Here is an example for BasicPayload object:
0232 run ./getPayloadData.py --plugin pluginBasicPayload_PayloadInspector --plot plot_BasicPayload_data0 --tag BasicPayload_v2 --time_type Run --iovs '{"start_iov": "201", "end_iov": "801"}' --db Prod --test
0233
0234 c) if it works correctly please make a pull request and once it's accepted
0235 go to cmsDbBrowser and wait for the next IB to test it.
0236 '''
0237
0238 parser = ArgumentParser(description=description, formatter_class=RawTextHelpFormatter)
0239 parser.add_argument("-d", "--discover", help="discovers object types and plots \nfor a given cmssw release", action="store_true")
0240 parser.add_argument("-i", "--iovs", help="deserializes given iovs data encoded in base64 and returns plot coordinates also encoded in base64")
0241 parser.add_argument("-i2", "--iovstwo", help="deserializes given iovs data encoded in base64 and returns plot coordinates also encoded in base64")
0242 parser.add_argument("-o", "--plugin", help="Payload Inspector plugin name needed for iovs deserialization")
0243 parser.add_argument("-p", "--plot", help="plot name needed for iovs deserialization")
0244 parser.add_argument("-t", "--tag", help="tag name needed for iovs deserialization")
0245 parser.add_argument("-t2", "--tagtwo", help="tag name needed for iovs deserialization")
0246 parser.add_argument("-tt", "--time_type", help="tag time type name needed for iovs deserialization")
0247 parser.add_argument("-b", "--db", help="db (Prod or Prep) needed for iovs deserialization")
0248 parser.add_argument("-test", "--test", help="add this flag if you want to test the deserialization function and want to see a readable output", action="store_true")
0249 parser.add_argument("-v", "--verbose", help="verbose mode. Shows more information", action="store_true")
0250 parser.add_argument("-ip","--image_plot", help="Switch telling the script that this plot type is of type Image", action="store_true")
0251 parser.add_argument("-s", "--suppress-output", help="Supresses output from so that stdout and stderr can be kept pure for the ssh transmission", action="store_true")
0252 parser.add_argument("-is", "--input_params", help="Plot input parameters ( dictionary, JSON serialized into string )" )
0253
0254
0255 if len(sys.argv) == 1:
0256 parser.print_help()
0257 sys.exit(1)
0258
0259 args = parser.parse_args()
0260
0261
0262 if args.discover:
0263 os.write( 1, str.encode(discover()) )
0264
0265 input_params = None
0266 if args.input_params is not None:
0267 input_params = yaml.safe_load(args.input_params)
0268
0269 tags = []
0270 if args.tag:
0271 iovDict = yaml.safe_load( args.iovs )
0272 tags.append( (args.tag, iovDict['start_iov'], iovDict['end_iov'] ) )
0273 if args.tagtwo:
0274 iovDict = yaml.safe_load( args.iovstwo )
0275 tags.append( (args.tagtwo, iovDict['start_iov'], iovDict['end_iov'] ) )
0276
0277 result = deserialize_iovs(args.db, args.plugin, args.plot, tags, args.time_type, input_params)
0278
0279
0280 if args.test:
0281 os.write( 2, str.encode(json.dumps( json.loads( result ), indent=4 )))
0282
0283
0284 elif args.image_plot:
0285
0286 filename = None
0287
0288 try:
0289 filename = json.loads( result )['file']
0290
0291 except ValueError as e:
0292 os.write( 2, 'Value error when getting image name: %s\n' % str( e ))
0293 except KeyError as e:
0294 os.write( 2, 'Key error when getting image name: %s\n' % str( e ))
0295
0296 if not filename or not os.path.isfile( filename ):
0297 os.write( 2, str.encode('Error: Generated image file (%s) not found\n' % filename ))
0298
0299 try:
0300 with open( filename, 'rb' ) as f:
0301 shutil.copyfileobj( f, sys.stdout.buffer )
0302 except IOError as e:
0303 os.write( 2, str.encode('IO error when streaming image: %s' % str( e )))
0304 finally:
0305 os.remove( filename )
0306
0307
0308
0309 else:
0310 import base64
0311 os.write( 1, base64.b64encode(bytes(result, 'utf-8')))