Utilities/scripts/getPayloadData.py

0001 #!/usr/bin/env python3
0002
0003
0004 from __future__ import print_function
0005 import shutil
0006 import glob
0007 import json
0008 import yaml
0009 import sys
0010 import sys
0011 import os
0012
0013 from importlib  import import_module
0014 from argparse   import ArgumentParser, RawTextHelpFormatter
0015
0016
0017 import pluginCondDBV2PyInterface
0018 pluginCondDBV2PyInterface.CMSSWInit()
0019
0020
0021
0022 def supress_output( f ):
0023     '''
0024     Temporarily disables stdout and stderr so that printouts from the plot
0025     plugin does not compromise the purity of our ssh stream if
0026     args.suppress_output is true
0027     '''
0028     def decorated( *fargs, **fkwargs ):
0029
0030         suppress = args.suppress_output
0031         if suppress:
0032
0033             # Get rid of what is already there ( should be nothing for this script )
0034             sys.stdout.flush()
0035
0036             # Save file descriptors so it can be reactivated later
0037             saved_stdout = os.dup( 1 )
0038             saved_stderr = os.dup( 2 )
0039
0040             # /dev/null is used just to discard what is being printed
0041             devnull = os.open( '/dev/null', os.O_WRONLY )
0042
0043             # Duplicate the file descriptor for /dev/null
0044             # and overwrite the value for stdout (file descriptor 1)
0045             os.dup2( devnull, 1 )
0046             os.dup2( devnull, 2 )
0047
0048         result = f( *fargs, **fkwargs )
0049
0050         if suppress:
0051
0052             # Close devnull after duplication (no longer needed)
0053             os.close( devnull )
0054
0055             # Reenable stdout and stderr
0056             os.dup2( saved_stdout, 1 )
0057             os.dup2( saved_stderr, 2 )
0058
0059         return result
0060
0061     return decorated
0062
0063
0064 @supress_output
0065 def deserialize_iovs(db, plugin_name, plot_name, tags, time_type, input_params):
0066     ''' Deserializes given iovs data and returns plot coordinates '''
0067
0068     output('Starting to deserialize iovs: ', '')
0069     output('db: ', db)
0070     output('plugin name: ', plugin_name)
0071     output('plot name: ', plot_name)
0072     output('tags: ', tags)
0073     output('tag time type: ', time_type)
0074
0075     plugin_base = import_module('pluginModule_PayloadInspector')
0076     output('PI plugin base: ', plugin_base)
0077
0078     plugin_obj = import_module(plugin_name)
0079     output('PI plugin object: ', plugin_obj)
0080
0081     # get plot method and execute it with given iovs
0082     plot = getattr(plugin_obj, plot_name)()
0083     output('plot object: ', plot)
0084
0085     if db == "Prod":
0086         db_name = 'frontier://FrontierProd/CMS_CONDITIONS'
0087     elif db == 'Prep' :
0088         db_name = 'frontier://FrontierPrep/CMS_CONDITIONS'
0089     else:
0090         db_name = db
0091
0092     output('full DB name: ', db_name)
0093
0094     if input_params is not None:
0095         plot.setInputParamValues( input_params )
0096
0097     modv = getattr(plugin_base,'ModuleVersion')
0098
0099     success = False
0100     if modv.label == '1.0':
0101         if len(tags)==1:
0102             success = plot.process(db_name, tags[0][0], time_type, int(tags[0][1]), int(tags[0][2]) )
0103         elif len(tags)==2:
0104             success = plot.processTwoTags(db_name, tags[0][0], tags[1][0], int(tags[0][1]),int(tags[1][1]) )
0105     elif modv.label == '2.0':
0106         success = plot.process(db_name, tags)
0107
0108     output('plot processed data successfully: ', success)
0109     if not success:
0110         return False
0111
0112     result = plot.data()
0113     output('deserialized data: ', result)
0114     return result
0115
0116 def discover_plugins():
0117     ''' Returns a list of Payload Inspector plugin names
0118         Example:
0119         ['pluginBasicPayload_PayloadInspector', 'pluginBeamSpot_PayloadInspector', 'pluginSiStrip_PayloadInspector']
0120     '''
0121     architecture = os.environ.get('SCRAM_ARCH', None)
0122     output('architecture: ', architecture)
0123
0124     plugins = []
0125     releases = [
0126         os.environ.get('CMSSW_BASE', None),
0127         os.environ.get('CMSSW_RELEASE_BASE', None),
0128         os.environ.get('CMSSW_FULL_RELEASE_BASE', None)
0129     ]
0130
0131     for r in releases:
0132         if not r: continue # skip if release base is not specified
0133         output('* release: ', r)
0134
0135         path = os.path.join(r, 'lib', architecture)
0136         output('* full release path: ', path)
0137
0138         plugins += glob.glob(path + '/plugin*_PayloadInspector.so' )
0139         output('found plugins: ', plugins)
0140
0141         # If no plugins are found in the local release,
0142         # go find them in the release base (or full release base, in case of patches)
0143         if(len(plugins)==0):
0144             output('# plugins found:',len(plugins))
0145         else:
0146             if r: break # break loop if CMSSW_BASE is specified
0147
0148     # extracts the object name from plugin path:
0149     # /afs/cern.ch/cms/slc6_amd64_gcc493/cms/cmssw/CMSSW_8_0_6/lib/slc6_amd64_gcc493/pluginBasicPayload_PayloadInspector.so
0150     # becomes pluginBasicPayload_PayloadInspector
0151     result = []
0152     for p in plugins:
0153          result.append(p.split('/')[-1].replace('.so', ''))
0154
0155     output('discovered plugins: ', result)
0156     return result
0157
0158 def discover():
0159     ''' Discovers object types and plots for a given cmssw release
0160         Example:
0161         {
0162             "BasicPayload": [
0163                 {"plot": "plot_BeamSpot_x", "plot_type": "History",
0164                  "single_iov": false, "plugin_name": "pluginBeamSpot_PayloadInspector",
0165                  "title": "x vs run number"},
0166                 ...
0167             ],
0168            ...
0169         }
0170     '''
0171     plugin_base = import_module('pluginModule_PayloadInspector')
0172     modv = getattr(plugin_base,'ModuleVersion')
0173     result = {}
0174     for plugin_name in discover_plugins():
0175         output(' - plugin name: ', plugin_name)
0176         plugin_obj = import_module(plugin_name)
0177         output('*** PI plugin object: ', plugin_obj)
0178         for plot in dir(plugin_obj):
0179             if 'plot_' not in plot: continue # skip if method doesn't start with 'plot_' prefix
0180             output(' - plot name: ', plot)
0181             plot_method= getattr(plugin_obj, plot)()
0182             output(' - plot object: ', plot_method)
0183             payload_type = plot_method.payloadType()
0184             output(' - payload type: ', payload_type)
0185             plot_title = plot_method.title()
0186             output(' - plot title: ', plot_title)
0187             plot_type = plot_method.type()
0188             output(' - plot type: ', plot_type)
0189             single_iov = plot_method.isSingleIov()
0190             output(' - is single iov: ', single_iov)
0191             two_tags = plot_method.isTwoTags()
0192             output(' - is Two Tags: ', two_tags)
0193             plot_dict = { 'plot': plot, 'plugin_name': plugin_name, 'title': plot_title, 'plot_type': plot_type, 'single_iov': single_iov, 'two_tags': two_tags }
0194             if modv.label == '2.0':
0195                 input_params = plot_method.inputParams()
0196                 output(' - input params: ', len(input_params))
0197                 plot_dict[ 'input_params'] = input_params
0198             result.setdefault(payload_type, []).append( plot_dict )
0199             output('currently discovered info: ', result)
0200     output('*** final output:', '')
0201     return json.dumps(result)
0202
0203 def output(description, param):
0204     if args.verbose:
0205         print('')
0206         print(description, param)
0207
0208 if __name__ == '__main__':
0209
0210     description = '''
0211     Payload Inspector - data visualisation tool which is integrated into the cmsDbBrowser.
0212     It allows to display plots and monitor the calibration and alignment data.
0213
0214     You can access Payload Inspector with a link below:
0215     https://cms-conddb.cern.ch/cmsDbBrowser/payload_inspector/Prod
0216
0217     This script is a part of the Payload Inspector service and is responsible for:
0218     a) discovering PI objects that are available in a given cmssw release
0219     b) deserializing payload data which is later used as plot coordinates
0220     c) testing new PI objects which are under development
0221
0222     To test new PI objects please do the following:
0223     a) run ./getPayloadData.py --discover
0224     to check if your newly created object is found by the script.
0225     Please note that we strongly rely on naming conventions so if you don't
0226     see your object listed you probably misnamed it in objectType() method.
0227     Also all plot methods should start with "plot_" prefix.
0228
0229     b) second step is to test if it returns data correctly:
0230     run ./getPayloadData.py --plugin YourPIPluginName --plot YourObjectPlot --tag tagName --time_type Run --iovs '{"start_iov": "201", "end_iov": "801"}' --db Prod --test
0231
0232     Here is an example for BasicPayload object:
0233     run ./getPayloadData.py --plugin pluginBasicPayload_PayloadInspector --plot plot_BasicPayload_data0 --tag BasicPayload_v2 --time_type Run --iovs '{"start_iov": "201", "end_iov": "801"}' --db Prod --test
0234
0235     c) if it works correctly please make a pull request and once it's accepted
0236     go to cmsDbBrowser and wait for the next IB to test it.
0237     '''
0238
0239     parser = ArgumentParser(description=description, formatter_class=RawTextHelpFormatter)
0240     parser.add_argument("-d", "--discover",   help="discovers object types and plots \nfor a given cmssw release", action="store_true")
0241     parser.add_argument("-i", "--iovs",       help="deserializes given iovs data encoded in base64 and returns plot coordinates also encoded in base64")
0242     parser.add_argument("-i2", "--iovstwo",   help="deserializes given iovs data encoded in base64 and returns plot coordinates also encoded in base64")
0243     parser.add_argument("-o", "--plugin",     help="Payload Inspector plugin name needed for iovs deserialization")
0244     parser.add_argument("-p", "--plot",       help="plot name needed for iovs deserialization")
0245     parser.add_argument("-t", "--tag",        help="tag name needed for iovs deserialization")
0246     parser.add_argument("-t2", "--tagtwo",    help="tag name needed for iovs deserialization")
0247     parser.add_argument("-tt", "--time_type", help="tag time type name needed for iovs deserialization")
0248     parser.add_argument("-b", "--db",         help="db (Prod or Prep) needed for iovs deserialization")
0249     parser.add_argument("-test", "--test",    help="add this flag if you want to test the deserialization function and want to see a readable output", action="store_true")
0250     parser.add_argument("-v", "--verbose",    help="verbose mode. Shows more information", action="store_true")
0251     parser.add_argument("-ip","--image_plot", help="Switch telling the script that this plot type is of type Image", action="store_true")
0252     parser.add_argument("-s", "--suppress-output", help="Supresses output from so that stdout and stderr can be kept pure for the ssh transmission", action="store_true")
0253     parser.add_argument("-is", "--input_params", help="Plot input parameters ( dictionary, JSON serialized into string )" )
0254
0255     # shows help if no arguments are provided
0256     if len(sys.argv) == 1:
0257         parser.print_help()
0258         sys.exit(1)
0259
0260     args = parser.parse_args()
0261
0262     # Return discover of plot if requested
0263     if args.discover:
0264         os.write( 1, str.encode(discover()) )
0265
0266     input_params = None
0267     if args.input_params is not None:
0268         input_params = yaml.safe_load(args.input_params)
0269
0270     tags = []
0271     if args.tag:
0272         iovDict = yaml.safe_load( args.iovs )
0273         tags.append( (args.tag, iovDict['start_iov'], iovDict['end_iov'] ) )
0274         if args.tagtwo:
0275             iovDict = yaml.safe_load( args.iovstwo )
0276             tags.append( (args.tagtwo, iovDict['start_iov'], iovDict['end_iov'] ) )
0277
0278         result = deserialize_iovs(args.db, args.plugin, args.plot, tags, args.time_type, input_params)
0279
0280         # If test -> output the result as formatted json
0281         if args.test:
0282             os.write( 2, str.encode(json.dumps( json.loads( result ), indent=4 )))
0283
0284         # If image plot -> get image file from result, open it and output bytes
0285         elif args.image_plot:
0286
0287             filename = None
0288
0289             try:
0290                 filename = json.loads( result )['file']
0291                 #print 'File name',filename
0292             except ValueError as e:
0293                 os.write( 2, 'Value error when getting image name: %s\n' % str( e ))
0294             except KeyError as e:
0295                 os.write( 2, 'Key error when getting image name: %s\n' % str( e ))
0296
0297             if not filename or not os.path.isfile( filename ):
0298                 os.write( 2, str.encode('Error: Generated image file (%s) not found\n' % filename ))
0299
0300             try:
0301                 with open( filename, 'rb' ) as f:
0302                     shutil.copyfileobj( f, sys.stdout.buffer )
0303             except IOError as e:
0304                 os.write( 2, str.encode('IO error when streaming image: %s' % str( e )))
0305             finally:
0306                 os.remove( filename )
0307
0308
0309         # Else -> output result json string with base 64 encoding
0310         else:
0311             import base64
0312             os.write( 1, base64.b64encode(bytes(result, 'utf-8')))