Utilities/python/cond2xml.py

0001 from __future__ import print_function
0002
0003 import os
0004 import shutil
0005 import sys
0006 import time
0007 import glob
0008 import importlib
0009 import logging
0010 import subprocess
0011
0012 # as we need to load the shared lib from here, make sure it's in our path:
0013 if os.path.join( os.environ['CMSSW_BASE'], 'src') not in sys.path:
0014     sys.path.append( os.path.join( os.environ['CMSSW_BASE'], 'src') )
0015
0016 # -------------------------------------------------------------------------------------------------------
0017
0018 payload2xmlCodeTemplate = """
0019
0020 #include "CondCore/Utilities/interface/Payload2XMLModule.h"
0021 #include "CondCore/Utilities/src/CondFormats.h"
0022
0023 PAYLOAD_2XML_MODULE( %s ){
0024   PAYLOAD_2XML_CLASS( %s );
0025 }
0026
0027 """
0028
0029 buildFileTemplate = """
0030 <flags CXXFLAGS="-Wno-sign-compare -Wno-unused-variable -Os"/>
0031 <library   file="%s" name="%s">
0032   <use   name="CondCore/Utilities"/>
0033   <use name="py3-pybind11"/>
0034   <use name="python3"/>
0035 </library>
0036 <export>
0037   <lib   name="1"/>
0038 </export>
0039 """
0040
0041 # helper function
0042 def sanitize(typeName):
0043     return typeName.replace(' ','').replace('<','_').replace('>','')
0044
0045 def localLibName( payloadType ):
0046     # required to avoid ( unlikely ) clashes between lib names from templates and lib names from classes
0047     prefix = ''
0048     if '<' in payloadType and '>' in payloadType:
0049         prefix = 't'
0050     ptype = payloadType
0051     if '::' in payloadType:
0052         ptype = payloadType.replace('::','_')
0053     return "%s_%spayload2xml" %(sanitize(ptype),prefix)
0054
0055 def boost_version_for_this_release():
0056     import pluginUtilities_payload2xml as mod2XML
0057     return mod2XML.boost_version_label()
0058
0059 class CondXmlProcessor(object):
0060
0061     def __init__(self, condDBIn):
0062         self.conddb = condDBIn
0063
0064         if not os.path.exists( os.path.join( os.environ['CMSSW_BASE'], 'src') ):
0065             raise Exception("Looks like you are not running in a CMSSW developer area, $CMSSW_BASE/src/ does not exist")
0066
0067         self.fakePkgName = "fakeSubSys4pl/fakePkg4pl"
0068         self._pl2xml_tmpDir = os.path.join( os.environ['CMSSW_BASE'], 'src', self.fakePkgName )
0069
0070         self.doCleanup = False
0071
0072     def __del__(self):
0073
0074         if self.doCleanup:
0075             shutil.rmtree( '/'.join( self._pl2xml_tmpDir.split('/')[:-1] ) )
0076         return
0077
0078     def discover(self, payloadType):
0079
0080         libName = 'pluginUtilities_payload2xml.so'
0081         # first search: developer area or main release
0082         libDir = os.path.join( os.environ["CMSSW_BASE"], 'lib', os.environ["SCRAM_ARCH"] )
0083         devLibDir = libDir
0084         libPath = os.path.join( devLibDir, libName )
0085         releaseBase = os.environ["CMSSW_RELEASE_BASE"]
0086         devCheckout = (releaseBase != '')
0087         if not devCheckout:
0088             logging.debug('Looks like the current working environment is a read-only release')
0089         if not os.path.exists( libPath ) and devCheckout:
0090             # main release ( for dev checkouts )
0091             libDir = os.path.join( releaseBase, 'lib', os.environ["SCRAM_ARCH"] )
0092             libPath = os.path.join( libDir, libName )
0093             if not os.path.exists( libPath ):
0094                 if "CMSSW_FULL_RELEASE_BASE" in os.environ:
0095                     libDir = os.path.join( os.environ["CMSSW_FULL_RELEASE_BASE"], 'lib', os.environ["SCRAM_ARCH"] )
0096                     libPath = os.path.join( libDir, libName )
0097         if not os.path.exists( libPath ):
0098             # it should never happen!
0099             raise Exception('No built-in library %s found with XML converters.' %libPath)
0100         logging.debug("Importing built-in library %s" %libPath)
0101         module = importlib.import_module( libName.replace('.so', '') )
0102         functors = dir(module)
0103         funcName = payloadType+'2xml'
0104         if funcName in functors:
0105             logging.info('XML converter for payload class %s found in the built-in library.' %payloadType)
0106             return getattr( module, funcName)
0107         if not devCheckout:
0108             # give-up if it is a read-only release...
0109             raise Exception('No XML converter suitable for payload class %s has been found in the built-in library.')
0110         libName = 'plugin%s.so' %localLibName( payloadType )
0111         libPath = os.path.join( devLibDir, libName )
0112         if os.path.exists( libPath ):
0113             logging.info('Found local library with XML converter for class %s' %payloadType )
0114             module = importlib.import_module( libName.replace('.so', '') )
0115             return getattr( module, funcName)
0116         logging.warning('No XML converter for payload class %s found in the built-in library.' %payloadType)
0117         return None
0118
0119     def prepPayload2xml(self, payloadType):
0120
0121         converter = self.discover(payloadType)
0122         if converter: return converter
0123
0124         #otherwise, go for the code generation in the local checkout area.
0125         startTime = time.time()
0126
0127         libName = localLibName( payloadType )
0128         pluginName = 'plugin%s' % libName
0129         tmpLibName = "Tmp_payload2xml"
0130         tmpPluginName = 'plugin%s' %tmpLibName
0131
0132         libDir = os.path.join( os.environ["CMSSW_BASE"], 'lib', os.environ["SCRAM_ARCH"] )
0133         tmpLibFile = os.path.join( libDir,tmpPluginName+'.so' )
0134         code = payload2xmlCodeTemplate %(pluginName,payloadType)
0135
0136         tmpSrcFileName = 'Local_2XML.cpp'
0137         tmpDir = self._pl2xml_tmpDir
0138         if ( os.path.exists( tmpDir ) ) :
0139             msg = '\nERROR: %s already exists, please remove if you did not create that manually !!' % tmpDir
0140             raise Exception(msg)
0141
0142         logging.debug('Creating temporary package %s' %self._pl2xml_tmpDir)
0143         os.makedirs( tmpDir+'/plugins' )
0144
0145         buildFileName = "%s/plugins/BuildFile.xml" % (tmpDir,)
0146         with open(buildFileName, 'w') as buildFile:
0147             buildFile.write( buildFileTemplate %(tmpSrcFileName,tmpLibName) )
0148             buildFile.close()
0149
0150         tmpSrcFilePath = "%s/plugins/%s" % (tmpDir, tmpSrcFileName,)
0151         with open(tmpSrcFilePath, 'w') as codeFile:
0152             codeFile.write(code)
0153             codeFile.close()
0154
0155         cmd = "source $CMS_PATH/cmsset_default.sh;"
0156         cmd += "(cd %s ; scram b 2>&1 >build.log)" %tmpDir
0157         pipe = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
0158         out, err = pipe.communicate()
0159         ret = pipe.returncode
0160
0161         buildTime = time.time()-startTime
0162         logging.info("Building done in %s sec., return code from build: %s" %(buildTime,ret) )
0163
0164         if (ret != 0):
0165             logging.error("Local build for xml dump failed.")
0166             return None
0167
0168         libFile = os.path.join(libDir,pluginName + '.so')
0169         shutil.copyfile(tmpLibFile,libFile)
0170
0171         module =  importlib.import_module( pluginName )
0172         funcName = payloadType+'2xml'
0173         functor = getattr( module, funcName )
0174         self.doCleanup = True
0175         return functor
0176
0177     def payload2xml(self, session, payloadHash, destFile):
0178
0179         Payload = session.get_dbtype(self.conddb.Payload)
0180         # get payload from DB:
0181         result = session.query(Payload.data, Payload.object_type).filter(Payload.hash == payloadHash).one()
0182         data, plType = result
0183         logging.info('Found payload of type %s' %plType)
0184
0185         convFuncName = sanitize(plType)+'2xml'
0186         xmlConverter = self.prepPayload2xml(plType)
0187
0188         if xmlConverter is not None:
0189             obj = xmlConverter()
0190             resultXML = obj.write( data )
0191             if destFile is None:
0192                 print(resultXML)
0193             else:
0194                 with open(destFile, 'w') as outFile:
0195                     outFile.write(resultXML)
0196                     outFile.close()