Utilities/python/cond2xml.py

0001
0002 import os
0003 import shutil
0004 import sys
0005 import time
0006 import glob
0007 import importlib
0008 import logging
0009 import subprocess
0010
0011 # as we need to load the shared lib from here, make sure it's in our path:
0012 if os.path.join( os.environ['CMSSW_BASE'], 'src') not in sys.path:
0013     sys.path.append( os.path.join( os.environ['CMSSW_BASE'], 'src') )
0014
0015 # -------------------------------------------------------------------------------------------------------
0016
0017 payload2xmlCodeTemplate = """
0018
0019 #include "CondCore/Utilities/interface/Payload2XMLModule.h"
0020 #include "CondCore/Utilities/src/CondFormats.h"
0021
0022 PAYLOAD_2XML_MODULE( %s ){
0023   PAYLOAD_2XML_CLASS( %s );
0024 }
0025
0026 """
0027
0028 buildFileTemplate = """
0029 <flags CXXFLAGS="-Wno-sign-compare -Wno-unused-variable -Os"/>
0030 <library   file="%s" name="%s">
0031   <use   name="CondCore/Utilities"/>
0032   <use name="py3-pybind11"/>
0033   <use name="python3"/>
0034 </library>
0035 <export>
0036   <lib   name="1"/>
0037 </export>
0038 """
0039
0040 # helper function
0041 def sanitize(typeName):
0042     return typeName.replace(' ','').replace('<','_').replace('>','')
0043
0044 def localLibName( payloadType ):
0045     # required to avoid ( unlikely ) clashes between lib names from templates and lib names from classes
0046     prefix = ''
0047     if '<' in payloadType and '>' in payloadType:
0048         prefix = 't'
0049     ptype = payloadType
0050     if '::' in payloadType:
0051         ptype = payloadType.replace('::','_')
0052     return "%s_%spayload2xml" %(sanitize(ptype),prefix)
0053
0054 def boost_version_for_this_release():
0055     import pluginUtilities_payload2xml as mod2XML
0056     return mod2XML.boost_version_label()
0057
0058 class CondXmlProcessor(object):
0059
0060     def __init__(self, condDBIn):
0061         self.conddb = condDBIn
0062
0063         if not os.path.exists( os.path.join( os.environ['CMSSW_BASE'], 'src') ):
0064             raise Exception("Looks like you are not running in a CMSSW developer area, $CMSSW_BASE/src/ does not exist")
0065
0066         self.fakePkgName = "fakeSubSys4pl/fakePkg4pl"
0067         self._pl2xml_tmpDir = os.path.join( os.environ['CMSSW_BASE'], 'src', self.fakePkgName )
0068
0069         self.doCleanup = False
0070
0071     def __del__(self):
0072
0073         if self.doCleanup:
0074             shutil.rmtree( '/'.join( self._pl2xml_tmpDir.split('/')[:-1] ) )
0075         return
0076
0077     def discover(self, payloadType):
0078         libName = 'pluginUtilities_payload2xml.so'
0079
0080         isReadOnlyRel = (os.environ['CMSSW_RELEASE_BASE'] == '')
0081         if isReadOnlyRel:
0082             logging.debug('Looks like the current working environment is a read-only release')
0083
0084         # first search CMSSW_BASE (developer area), then CMSSW_RELEASE_BASE (release base),
0085         # and finally CMSSW_FULL_RELEASE_BASE (full release base, defined only for patch releases)
0086         foundLib = False
0087         for cmsswBase in ['CMSSW_BASE', 'CMSSW_RELEASE_BASE', 'CMSSW_FULL_RELEASE_BASE']:
0088             # Skip to next in case one is not defined or is empty
0089             if not (cmsswBase in os.environ and os.environ[cmsswBase] != ''):
0090                 continue
0091             libDir = os.path.join( os.environ[cmsswBase], 'lib', os.environ['SCRAM_ARCH'] )
0092             libPath = os.path.join( libDir, libName )
0093             if cmsswBase == 'CMSSW_BASE':
0094                 devLibDir = libDir
0095             foundLib = os.path.isfile( libPath )
0096             if foundLib:
0097                 logging.debug('Found built-in library with XML converters: %s' %libPath)
0098                 break
0099         if not foundLib:
0100             # this should never happen !!
0101             raise Exception('No built-in library found with XML converters (library name: %s).' %libName)
0102
0103         logging.debug('Importing built-in library %s' %libPath)
0104         module = importlib.import_module( libName.replace('.so', '') )
0105         functors = dir(module)
0106         funcName = payloadType+'2xml'
0107         if funcName in functors:
0108             logging.info('XML converter for payload class %s found in the built-in library.' %payloadType)
0109             return getattr( module, funcName)
0110         if isReadOnlyRel:
0111             # give up if it is a read-only release...
0112             raise Exception('No XML converter suitable for payload class %s has been found in the built-in library.' %payloadType)
0113         libName = 'plugin%s.so' %localLibName( payloadType )
0114         libPath = os.path.join( devLibDir, libName )
0115         if os.path.exists( libPath ):
0116             logging.info('Found local library with XML converter for class %s' %payloadType)
0117             module = importlib.import_module( libName.replace('.so', '') )
0118             return getattr( module, funcName)
0119         logging.warning('No XML converter for payload class %s found in the built-in library.' %payloadType)
0120         return None
0121
0122     def prepPayload2xml(self, payloadType):
0123
0124         converter = self.discover(payloadType)
0125         if converter: return converter
0126
0127         #otherwise, go for the code generation in the local checkout area.
0128         startTime = time.time()
0129
0130         libName = localLibName( payloadType )
0131         pluginName = 'plugin%s' % libName
0132         tmpLibName = "Tmp_payload2xml"
0133         tmpPluginName = 'plugin%s' %tmpLibName
0134
0135         libDir = os.path.join( os.environ["CMSSW_BASE"], 'lib', os.environ["SCRAM_ARCH"] )
0136         tmpLibFile = os.path.join( libDir,tmpPluginName+'.so' )
0137         code = payload2xmlCodeTemplate %(pluginName,payloadType)
0138
0139         tmpSrcFileName = 'Local_2XML.cpp'
0140         tmpDir = self._pl2xml_tmpDir
0141         if ( os.path.exists( tmpDir ) ) :
0142             msg = '\nERROR: %s already exists, please remove if you did not create that manually !!' % tmpDir
0143             raise Exception(msg)
0144
0145         logging.debug('Creating temporary package %s' %self._pl2xml_tmpDir)
0146         os.makedirs( tmpDir+'/plugins' )
0147
0148         buildFileName = "%s/plugins/BuildFile.xml" % (tmpDir,)
0149         with open(buildFileName, 'w') as buildFile:
0150             buildFile.write( buildFileTemplate %(tmpSrcFileName,tmpLibName) )
0151             buildFile.close()
0152
0153         tmpSrcFilePath = "%s/plugins/%s" % (tmpDir, tmpSrcFileName,)
0154         with open(tmpSrcFilePath, 'w') as codeFile:
0155             codeFile.write(code)
0156             codeFile.close()
0157
0158         cmd = "source $CMS_PATH/cmsset_default.sh;"
0159         cmd += "(cd %s ; scram b 2>&1 >build.log)" %tmpDir
0160         pipe = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
0161         out, err = pipe.communicate()
0162         ret = pipe.returncode
0163
0164         buildTime = time.time()-startTime
0165         logging.info("Building done in %s sec., return code from build: %s" %(buildTime,ret) )
0166
0167         if (ret != 0):
0168             logging.error("Local build for xml dump failed.")
0169             return None
0170
0171         libFile = os.path.join(libDir,pluginName + '.so')
0172         shutil.copyfile(tmpLibFile,libFile)
0173
0174         module =  importlib.import_module( pluginName )
0175         funcName = payloadType+'2xml'
0176         functor = getattr( module, funcName )
0177         self.doCleanup = True
0178         return functor
0179
0180     def payload2xml(self, session, payloadHash, destFile):
0181
0182         Payload = session.get_dbtype(self.conddb.Payload)
0183         # get payload from DB:
0184         result = session.query(Payload.data, Payload.object_type).filter(Payload.hash == payloadHash).one()
0185         data, plType = result
0186         logging.info('Found payload of type %s' %plType)
0187
0188         convFuncName = sanitize(plType)+'2xml'
0189         xmlConverter = self.prepPayload2xml(plType)
0190
0191         if xmlConverter is not None:
0192             obj = xmlConverter()
0193             resultXML = obj.write( data )
0194             if destFile is None:
0195                 print(resultXML)
0196             else:
0197                 with open(destFile, 'a') as outFile:
0198                     outFile.write(resultXML)
0199