Utilities/python/cond2xml.py

0001 from __future__ import print_function
0002
0003 import os
0004 import shutil
0005 import sys
0006 import time
0007 import glob
0008 import importlib
0009 import logging
0010 import subprocess
0011
0012 # as we need to load the shared lib from here, make sure it's in our path:
0013 if os.path.join( os.environ['CMSSW_BASE'], 'src') not in sys.path:
0014     sys.path.append( os.path.join( os.environ['CMSSW_BASE'], 'src') )
0015
0016 # -------------------------------------------------------------------------------------------------------
0017
0018 payload2xmlCodeTemplate = """
0019
0020 #include "CondCore/Utilities/interface/Payload2XMLModule.h"
0021 #include "CondCore/Utilities/src/CondFormats.h"
0022
0023 PAYLOAD_2XML_MODULE( %s ){
0024   PAYLOAD_2XML_CLASS( %s );
0025 }
0026
0027 """
0028
0029 buildFileTemplate = """
0030 <flags CXXFLAGS="-Wno-sign-compare -Wno-unused-variable -Os"/>
0031 <library   file="%s" name="%s">
0032   <use   name="CondCore/Utilities"/>
0033   <use name="py3-pybind11"/>
0034   <use name="python3"/>
0035 </library>
0036 <export>
0037   <lib   name="1"/>
0038 </export>
0039 """
0040
0041 # helper function
0042 def sanitize(typeName):
0043     return typeName.replace(' ','').replace('<','_').replace('>','')
0044
0045 def localLibName( payloadType ):
0046     # required to avoid ( unlikely ) clashes between lib names from templates and lib names from classes
0047     prefix = ''
0048     if '<' in payloadType and '>' in payloadType:
0049         prefix = 't'
0050     ptype = payloadType
0051     if '::' in payloadType:
0052         ptype = payloadType.replace('::','_')
0053     return "%s_%spayload2xml" %(sanitize(ptype),prefix)
0054
0055 def boost_version_for_this_release():
0056     import pluginUtilities_payload2xml as mod2XML
0057     return mod2XML.boost_version_label()
0058
0059 class CondXmlProcessor(object):
0060
0061     def __init__(self, condDBIn):
0062         self.conddb = condDBIn
0063
0064         if not os.path.exists( os.path.join( os.environ['CMSSW_BASE'], 'src') ):
0065             raise Exception("Looks like you are not running in a CMSSW developer area, $CMSSW_BASE/src/ does not exist")
0066
0067         self.fakePkgName = "fakeSubSys4pl/fakePkg4pl"
0068         self._pl2xml_tmpDir = os.path.join( os.environ['CMSSW_BASE'], 'src', self.fakePkgName )
0069
0070         self.doCleanup = False
0071
0072     def __del__(self):
0073
0074         if self.doCleanup:
0075             shutil.rmtree( '/'.join( self._pl2xml_tmpDir.split('/')[:-1] ) )
0076         return
0077
0078     def discover(self, payloadType):
0079         libName = 'pluginUtilities_payload2xml.so'
0080
0081         isReadOnlyRel = (os.environ['CMSSW_RELEASE_BASE'] == '')
0082         if isReadOnlyRel:
0083             logging.debug('Looks like the current working environment is a read-only release')
0084
0085         # first search CMSSW_BASE (developer area), then CMSSW_RELEASE_BASE (release base),
0086         # and finally CMSSW_FULL_RELEASE_BASE (full release base, defined only for patch releases)
0087         foundLib = False
0088         for cmsswBase in ['CMSSW_BASE', 'CMSSW_RELEASE_BASE', 'CMSSW_FULL_RELEASE_BASE']:
0089             # Skip to next in case one is not defined or is empty
0090             if not (cmsswBase in os.environ and os.environ[cmsswBase] != ''):
0091                 continue
0092             libDir = os.path.join( os.environ[cmsswBase], 'lib', os.environ['SCRAM_ARCH'] )
0093             libPath = os.path.join( libDir, libName )
0094             if cmsswBase == 'CMSSW_BASE':
0095                 devLibDir = libDir
0096             foundLib = os.path.isfile( libPath )
0097             if foundLib:
0098                 logging.debug('Found built-in library with XML converters: %s' %libPath)
0099                 break
0100         if not foundLib:
0101             # this should never happen !!
0102             raise Exception('No built-in library found with XML converters (library name: %s).' %libName)
0103
0104         logging.debug('Importing built-in library %s' %libPath)
0105         module = importlib.import_module( libName.replace('.so', '') )
0106         functors = dir(module)
0107         funcName = payloadType+'2xml'
0108         if funcName in functors:
0109             logging.info('XML converter for payload class %s found in the built-in library.' %payloadType)
0110             return getattr( module, funcName)
0111         if isReadOnlyRel:
0112             # give up if it is a read-only release...
0113             raise Exception('No XML converter suitable for payload class %s has been found in the built-in library.' %payloadType)
0114         libName = 'plugin%s.so' %localLibName( payloadType )
0115         libPath = os.path.join( devLibDir, libName )
0116         if os.path.exists( libPath ):
0117             logging.info('Found local library with XML converter for class %s' %payloadType)
0118             module = importlib.import_module( libName.replace('.so', '') )
0119             return getattr( module, funcName)
0120         logging.warning('No XML converter for payload class %s found in the built-in library.' %payloadType)
0121         return None
0122
0123     def prepPayload2xml(self, payloadType):
0124
0125         converter = self.discover(payloadType)
0126         if converter: return converter
0127
0128         #otherwise, go for the code generation in the local checkout area.
0129         startTime = time.time()
0130
0131         libName = localLibName( payloadType )
0132         pluginName = 'plugin%s' % libName
0133         tmpLibName = "Tmp_payload2xml"
0134         tmpPluginName = 'plugin%s' %tmpLibName
0135
0136         libDir = os.path.join( os.environ["CMSSW_BASE"], 'lib', os.environ["SCRAM_ARCH"] )
0137         tmpLibFile = os.path.join( libDir,tmpPluginName+'.so' )
0138         code = payload2xmlCodeTemplate %(pluginName,payloadType)
0139
0140         tmpSrcFileName = 'Local_2XML.cpp'
0141         tmpDir = self._pl2xml_tmpDir
0142         if ( os.path.exists( tmpDir ) ) :
0143             msg = '\nERROR: %s already exists, please remove if you did not create that manually !!' % tmpDir
0144             raise Exception(msg)
0145
0146         logging.debug('Creating temporary package %s' %self._pl2xml_tmpDir)
0147         os.makedirs( tmpDir+'/plugins' )
0148
0149         buildFileName = "%s/plugins/BuildFile.xml" % (tmpDir,)
0150         with open(buildFileName, 'w') as buildFile:
0151             buildFile.write( buildFileTemplate %(tmpSrcFileName,tmpLibName) )
0152             buildFile.close()
0153
0154         tmpSrcFilePath = "%s/plugins/%s" % (tmpDir, tmpSrcFileName,)
0155         with open(tmpSrcFilePath, 'w') as codeFile:
0156             codeFile.write(code)
0157             codeFile.close()
0158
0159         cmd = "source $CMS_PATH/cmsset_default.sh;"
0160         cmd += "(cd %s ; scram b 2>&1 >build.log)" %tmpDir
0161         pipe = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
0162         out, err = pipe.communicate()
0163         ret = pipe.returncode
0164
0165         buildTime = time.time()-startTime
0166         logging.info("Building done in %s sec., return code from build: %s" %(buildTime,ret) )
0167
0168         if (ret != 0):
0169             logging.error("Local build for xml dump failed.")
0170             return None
0171
0172         libFile = os.path.join(libDir,pluginName + '.so')
0173         shutil.copyfile(tmpLibFile,libFile)
0174
0175         module =  importlib.import_module( pluginName )
0176         funcName = payloadType+'2xml'
0177         functor = getattr( module, funcName )
0178         self.doCleanup = True
0179         return functor
0180
0181     def payload2xml(self, session, payloadHash, destFile):
0182
0183         Payload = session.get_dbtype(self.conddb.Payload)
0184         # get payload from DB:
0185         result = session.query(Payload.data, Payload.object_type).filter(Payload.hash == payloadHash).one()
0186         data, plType = result
0187         logging.info('Found payload of type %s' %plType)
0188
0189         convFuncName = sanitize(plType)+'2xml'
0190         xmlConverter = self.prepPayload2xml(plType)
0191
0192         if xmlConverter is not None:
0193             obj = xmlConverter()
0194             resultXML = obj.write( data )
0195             if destFile is None:
0196                 print(resultXML)
0197             else:
0198                 with open(destFile, 'a') as outFile:
0199                     outFile.write(resultXML)
0200