Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2024-11-25 02:29:20

0001 #!/usr/bin/env python3
0002 
0003 
0004 import shutil
0005 import glob
0006 import json
0007 import yaml
0008 import sys
0009 import sys
0010 import os
0011 
0012 from importlib  import import_module
0013 from argparse   import ArgumentParser, RawTextHelpFormatter
0014 
0015 
0016 import pluginCondDBV2PyInterface
0017 pluginCondDBV2PyInterface.CMSSWInit()
0018 
0019 
0020 
0021 def supress_output( f ):
0022     '''
0023     Temporarily disables stdout and stderr so that printouts from the plot
0024     plugin does not compromise the purity of our ssh stream if 
0025     args.suppress_output is true
0026     '''
0027     def decorated( *fargs, **fkwargs ):
0028 
0029         suppress = args.suppress_output
0030         if suppress:
0031             
0032             # Get rid of what is already there ( should be nothing for this script )
0033             sys.stdout.flush()
0034 
0035             # Save file descriptors so it can be reactivated later 
0036             saved_stdout = os.dup( 1 )
0037             saved_stderr = os.dup( 2 )
0038 
0039             # /dev/null is used just to discard what is being printed
0040             devnull = os.open( '/dev/null', os.O_WRONLY )
0041 
0042             # Duplicate the file descriptor for /dev/null
0043             # and overwrite the value for stdout (file descriptor 1)
0044             os.dup2( devnull, 1 )
0045             os.dup2( devnull, 2 )
0046 
0047         result = f( *fargs, **fkwargs )
0048 
0049         if suppress:
0050 
0051             # Close devnull after duplication (no longer needed)
0052             os.close( devnull )
0053 
0054             # Reenable stdout and stderr
0055             os.dup2( saved_stdout, 1 )
0056             os.dup2( saved_stderr, 2 )
0057 
0058         return result
0059 
0060     return decorated
0061 
0062 
0063 @supress_output
0064 def deserialize_iovs(db, plugin_name, plot_name, tags, time_type, input_params):
0065     ''' Deserializes given iovs data and returns plot coordinates '''
0066     
0067     output('Starting to deserialize iovs: ', '')
0068     output('db: ', db)
0069     output('plugin name: ', plugin_name)
0070     output('plot name: ', plot_name)
0071     output('tags: ', tags)
0072     output('tag time type: ', time_type)
0073   
0074     plugin_base = import_module('pluginModule_PayloadInspector')
0075     output('PI plugin base: ', plugin_base)
0076 
0077     plugin_obj = import_module(plugin_name)
0078     output('PI plugin object: ', plugin_obj)
0079 
0080     # get plot method and execute it with given iovs
0081     plot = getattr(plugin_obj, plot_name)()
0082     output('plot object: ', plot)
0083 
0084     if db == "Prod":
0085         db_name = 'frontier://FrontierProd/CMS_CONDITIONS'
0086     elif db == 'Prep' :
0087         db_name = 'frontier://FrontierPrep/CMS_CONDITIONS'
0088     else:
0089         db_name = db
0090 
0091     output('full DB name: ', db_name)
0092 
0093     if input_params is not None:
0094         plot.setInputParamValues( input_params )
0095 
0096     modv = getattr(plugin_base,'ModuleVersion')
0097 
0098     success = False
0099     if modv.label == '1.0':
0100         if len(tags)==1:
0101             success = plot.process(db_name, tags[0][0], time_type, int(tags[0][1]), int(tags[0][2]) )
0102         elif len(tags)==2:
0103             success = plot.processTwoTags(db_name, tags[0][0], tags[1][0], int(tags[0][1]),int(tags[1][1]) )
0104     elif modv.label == '2.0':
0105         success = plot.process(db_name, tags)
0106 
0107     output('plot processed data successfully: ', success)
0108     if not success:
0109         return False
0110 
0111     result = plot.data()
0112     output('deserialized data: ', result)
0113     return result
0114 
0115 def discover_plugins():
0116     ''' Returns a list of Payload Inspector plugin names
0117         Example:
0118         ['pluginBasicPayload_PayloadInspector', 'pluginBeamSpot_PayloadInspector', 'pluginSiStrip_PayloadInspector']
0119     '''
0120     architecture = os.environ.get('SCRAM_ARCH', None)
0121     output('architecture: ', architecture)
0122 
0123     plugins = []
0124     releases = [
0125         os.environ.get('CMSSW_BASE', None),
0126         os.environ.get('CMSSW_RELEASE_BASE', None),
0127         os.environ.get('CMSSW_FULL_RELEASE_BASE', None)
0128     ]
0129 
0130     for r in releases:
0131         if not r: continue # skip if release base is not specified
0132         output('* release: ', r)
0133 
0134         path = os.path.join(r, 'lib', architecture)
0135         output('* full release path: ', path)
0136 
0137         plugins += glob.glob(path + '/plugin*_PayloadInspector.so' )
0138         output('found plugins: ', plugins) 
0139         
0140         # If no plugins are found in the local release,
0141         # go find them in the release base (or full release base, in case of patches)
0142         if(len(plugins)==0):
0143             output('# plugins found:',len(plugins))
0144         else:
0145             if r: break # break loop if CMSSW_BASE is specified
0146   
0147     # extracts the object name from plugin path:
0148     # /afs/cern.ch/cms/slc6_amd64_gcc493/cms/cmssw/CMSSW_8_0_6/lib/slc6_amd64_gcc493/pluginBasicPayload_PayloadInspector.so
0149     # becomes pluginBasicPayload_PayloadInspector
0150     result = []
0151     for p in plugins:
0152          result.append(p.split('/')[-1].replace('.so', ''))
0153 
0154     output('discovered plugins: ', result)
0155     return result
0156 
0157 def discover():
0158     ''' Discovers object types and plots for a given cmssw release
0159         Example:
0160         {
0161             "BasicPayload": [
0162                 {"plot": "plot_BeamSpot_x", "plot_type": "History", 
0163                  "single_iov": false, "plugin_name": "pluginBeamSpot_PayloadInspector",
0164                  "title": "x vs run number"},
0165                 ...
0166             ],
0167            ...
0168         }
0169     '''
0170     plugin_base = import_module('pluginModule_PayloadInspector') 
0171     modv = getattr(plugin_base,'ModuleVersion')
0172     result = {}
0173     for plugin_name in discover_plugins():
0174         output(' - plugin name: ', plugin_name)
0175         plugin_obj = import_module(plugin_name)
0176         output('*** PI plugin object: ', plugin_obj)
0177         for plot in dir(plugin_obj):
0178             if 'plot_' not in plot: continue # skip if method doesn't start with 'plot_' prefix
0179             output(' - plot name: ', plot)
0180             plot_method= getattr(plugin_obj, plot)()
0181             output(' - plot object: ', plot_method)
0182             payload_type = plot_method.payloadType()
0183             output(' - payload type: ', payload_type)
0184             plot_title = plot_method.title()
0185             output(' - plot title: ', plot_title)
0186             plot_type = plot_method.type()
0187             output(' - plot type: ', plot_type)
0188             single_iov = plot_method.isSingleIov()
0189             output(' - is single iov: ', single_iov)
0190             two_tags = plot_method.isTwoTags()
0191             output(' - is Two Tags: ', two_tags)
0192             plot_dict = { 'plot': plot, 'plugin_name': plugin_name, 'title': plot_title, 'plot_type': plot_type, 'single_iov': single_iov, 'two_tags': two_tags }
0193             if modv.label == '2.0':
0194                 input_params = plot_method.inputParams()
0195                 output(' - input params: ', len(input_params))
0196                 plot_dict[ 'input_params'] = input_params
0197             result.setdefault(payload_type, []).append( plot_dict )
0198             output('currently discovered info: ', result)
0199     output('*** final output:', '')
0200     return json.dumps(result)
0201 
0202 def output(description, param):
0203     if args.verbose:
0204         print('')
0205         print(description, param)
0206 
0207 if __name__ == '__main__':
0208 
0209     description = '''
0210     Payload Inspector - data visualisation tool which is integrated into the cmsDbBrowser.
0211     It allows to display plots and monitor the calibration and alignment data.
0212 
0213     You can access Payload Inspector with a link below:
0214     https://cms-conddb.cern.ch/cmsDbBrowser/payload_inspector/Prod
0215 
0216     This script is a part of the Payload Inspector service and is responsible for:
0217     a) discovering PI objects that are available in a given cmssw release
0218     b) deserializing payload data which is later used as plot coordinates
0219     c) testing new PI objects which are under development
0220 
0221     To test new PI objects please do the following:
0222     a) run ./getPayloadData.py --discover
0223     to check if your newly created object is found by the script.
0224     Please note that we strongly rely on naming conventions so if you don't
0225     see your object listed you probably misnamed it in objectType() method.
0226     Also all plot methods should start with "plot_" prefix.
0227 
0228     b) second step is to test if it returns data correctly:
0229     run ./getPayloadData.py --plugin YourPIPluginName --plot YourObjectPlot --tag tagName --time_type Run --iovs '{"start_iov": "201", "end_iov": "801"}' --db Prod --test 
0230   
0231     Here is an example for BasicPayload object:
0232     run ./getPayloadData.py --plugin pluginBasicPayload_PayloadInspector --plot plot_BasicPayload_data0 --tag BasicPayload_v2 --time_type Run --iovs '{"start_iov": "201", "end_iov": "801"}' --db Prod --test
0233 
0234     c) if it works correctly please make a pull request and once it's accepted
0235     go to cmsDbBrowser and wait for the next IB to test it.
0236     '''
0237 
0238     parser = ArgumentParser(description=description, formatter_class=RawTextHelpFormatter)
0239     parser.add_argument("-d", "--discover",   help="discovers object types and plots \nfor a given cmssw release", action="store_true")
0240     parser.add_argument("-i", "--iovs",       help="deserializes given iovs data encoded in base64 and returns plot coordinates also encoded in base64")
0241     parser.add_argument("-i2", "--iovstwo",   help="deserializes given iovs data encoded in base64 and returns plot coordinates also encoded in base64")
0242     parser.add_argument("-o", "--plugin",     help="Payload Inspector plugin name needed for iovs deserialization")
0243     parser.add_argument("-p", "--plot",       help="plot name needed for iovs deserialization")
0244     parser.add_argument("-t", "--tag",        help="tag name needed for iovs deserialization")
0245     parser.add_argument("-t2", "--tagtwo",    help="tag name needed for iovs deserialization")
0246     parser.add_argument("-tt", "--time_type", help="tag time type name needed for iovs deserialization")
0247     parser.add_argument("-b", "--db",         help="db (Prod or Prep) needed for iovs deserialization")
0248     parser.add_argument("-test", "--test",    help="add this flag if you want to test the deserialization function and want to see a readable output", action="store_true")
0249     parser.add_argument("-v", "--verbose",    help="verbose mode. Shows more information", action="store_true")
0250     parser.add_argument("-ip","--image_plot", help="Switch telling the script that this plot type is of type Image", action="store_true")
0251     parser.add_argument("-s", "--suppress-output", help="Supresses output from so that stdout and stderr can be kept pure for the ssh transmission", action="store_true")
0252     parser.add_argument("-is", "--input_params", help="Plot input parameters ( dictionary, JSON serialized into string )" )
0253 
0254     # shows help if no arguments are provided
0255     if len(sys.argv) == 1:
0256         parser.print_help()
0257         sys.exit(1)
0258 
0259     args = parser.parse_args()
0260     
0261     # Return discover of plot if requested
0262     if args.discover:
0263         os.write( 1, str.encode(discover()) )
0264 
0265     input_params = None
0266     if args.input_params is not None:
0267         input_params = yaml.safe_load(args.input_params)
0268 
0269     tags = []
0270     if args.tag:
0271         iovDict = yaml.safe_load( args.iovs )
0272         tags.append( (args.tag, iovDict['start_iov'], iovDict['end_iov'] ) )
0273         if args.tagtwo:
0274             iovDict = yaml.safe_load( args.iovstwo )
0275             tags.append( (args.tagtwo, iovDict['start_iov'], iovDict['end_iov'] ) )
0276 
0277         result = deserialize_iovs(args.db, args.plugin, args.plot, tags, args.time_type, input_params)            
0278         
0279         # If test -> output the result as formatted json
0280         if args.test:
0281             os.write( 2, str.encode(json.dumps( json.loads( result ), indent=4 )))
0282 
0283         # If image plot -> get image file from result, open it and output bytes 
0284         elif args.image_plot:
0285 
0286             filename = None
0287             
0288             try:
0289                 filename = json.loads( result )['file']
0290                 #print 'File name',filename
0291             except ValueError as e:
0292                 os.write( 2, 'Value error when getting image name: %s\n' % str( e ))
0293             except KeyError as e:
0294                 os.write( 2, 'Key error when getting image name: %s\n' % str( e ))
0295 
0296             if not filename or not os.path.isfile( filename ):
0297                 os.write( 2, str.encode('Error: Generated image file (%s) not found\n' % filename ))
0298 
0299             try:
0300                 with open( filename, 'rb' ) as f:
0301                     shutil.copyfileobj( f, sys.stdout.buffer )
0302             except IOError as e:
0303                 os.write( 2, str.encode('IO error when streaming image: %s' % str( e )))
0304             finally:
0305                 os.remove( filename )
0306 
0307                         
0308         # Else -> output result json string with base 64 encoding
0309         else:
0310             import base64
0311             os.write( 1, base64.b64encode(bytes(result, 'utf-8')))