File indexing completed on 2024-11-25 02:29:23
0001
0002 '''CMS Conditions DB Serialization generator.
0003
0004 Generates the non-intrusive serialization code required for the classes
0005 marked with the COND_SERIALIZABLE macro.
0006
0007 The code was taken from the prototype that did many other things as well
0008 (finding transients, marking serializable classes, etc.). After removing
0009 everything but what is required to build the serialization, the code was
0010 made more robust and cleaned up a bit to be integrated on the BoostIO IB.
0011 However, the code still needs to be restructured a bit more to improve
0012 readability (e.g. name some constants, use a template engine, ask for
0013 clang's bindings to be installed along clang itself, etc.).
0014 '''
0015
0016 __author__ = 'Miguel Ojeda'
0017 __copyright__ = 'Copyright 2014, CERN'
0018 __credits__ = ['Giacomo Govi', 'Miguel Ojeda', 'Andreas Pfeiffer']
0019 __license__ = 'Unknown'
0020 __maintainer__ = 'Miguel Ojeda'
0021 __email__ = 'mojedasa@cern.ch'
0022
0023
0024 import argparse
0025 import logging
0026 import os
0027 import re
0028 import subprocess
0029
0030 import clang.cindex
0031
0032 clang_version = None
0033
0034 headers_template = '''
0035 #include "{headers}"
0036
0037 #include <boost/serialization/base_object.hpp>
0038 #include <boost/serialization/nvp.hpp>
0039 #include <boost/serialization/export.hpp>
0040
0041 #include "CondFormats/Serialization/interface/Equal.h"
0042 #include "CondFormats/Serialization/interface/Instantiate.h"
0043
0044 '''
0045
0046 serialize_method_begin_template = '''template <class Archive>
0047 void {klass}::serialize(Archive & ar, const unsigned int)
0048 {{'''
0049
0050 serialize_method_base_object_template = ' ar & boost::serialization::make_nvp("{base_object_name_sanitised}", boost::serialization::base_object<{base_object_name}>(*this));'
0051
0052 serialize_method_member_template = ''' ar & boost::serialization::make_nvp("{member_name_sanitised}", {member_name});'''
0053
0054 serialize_method_end = '''}
0055 '''
0056
0057 instantiation_template = '''COND_SERIALIZATION_INSTANTIATE({klass});
0058 '''
0059
0060
0061 skip_namespaces = frozenset([
0062
0063 '',
0064
0065
0066 'std', 'boost', 'mpl_', 'boost_swap_impl',
0067
0068
0069 'ROOT', 'edm', 'ora', 'coral', 'CLHEP', 'Geom', 'HepGeom',
0070 ])
0071
0072 def is_definition_by_loc(node):
0073 if node.get_definition() is None:
0074 return False
0075 if node.location is None or node.get_definition().location is None:
0076 return False
0077 return node.location == node.get_definition().location
0078
0079 def is_serializable_class(node):
0080 for child in node.get_children():
0081 if child.spelling != 'serialize' or child.kind != clang.cindex.CursorKind.FUNCTION_TEMPLATE or is_definition_by_loc(child):
0082 continue
0083
0084 if [(x.spelling, x.kind, is_definition_by_loc(x), x.type.kind) for x in child.get_children()] != [
0085 ('Archive', clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER, True, clang.cindex.TypeKind.UNEXPOSED),
0086 ('ar', clang.cindex.CursorKind.PARM_DECL, True, clang.cindex.TypeKind.LVALUEREFERENCE),
0087 ('version', clang.cindex.CursorKind.PARM_DECL, True, clang.cindex.TypeKind.UINT),
0088 ]:
0089 continue
0090
0091 return True
0092
0093 return False
0094
0095
0096 def is_serializable_class_manual(node):
0097 for child in node.get_children():
0098 if child.spelling == 'cond_serialization_manual' and child.kind == clang.cindex.CursorKind.CXX_METHOD and not is_definition_by_loc(child):
0099 return True
0100
0101 return False
0102
0103
0104 def get_statement(node):
0105
0106
0107
0108 if node.extent.start.file is None:
0109 return None
0110
0111 filename = node.extent.start.file.name
0112 start = node.extent.start.offset
0113 end = node.extent.end.offset
0114
0115 with open(filename, 'rb') as fd:
0116 source = fd.read().decode('latin-1')
0117
0118 return source[start:source.find(';', end)]
0119
0120
0121 def get_basic_type_string(node):
0122 typekinds = {
0123 clang.cindex.TypeKind.BOOL: 'bool',
0124 clang.cindex.TypeKind.INT: 'int',
0125 clang.cindex.TypeKind.LONG: 'long',
0126 clang.cindex.TypeKind.UINT: 'unsigned int',
0127 clang.cindex.TypeKind.ULONG: 'unsigned long',
0128 clang.cindex.TypeKind.FLOAT: 'float',
0129 clang.cindex.TypeKind.DOUBLE: 'double',
0130 }
0131
0132 if node.type.kind not in typekinds:
0133 raise Exception('Not a known basic type.')
0134
0135 return typekinds[node.type.kind]
0136
0137
0138 def get_type_string(node):
0139 spelling = node.type.get_declaration().spelling
0140 if spelling is not None:
0141 return spelling
0142
0143 return get_basic_type_string(node)
0144
0145
0146 def get_serializable_classes_members(node, all_template_types=None, namespace='', only_from_path=None):
0147 if all_template_types is None:
0148 all_template_types = []
0149
0150 logging.debug('%s', (node.spelling, all_template_types, namespace))
0151 results = {}
0152 for child in node.get_children():
0153 if child.kind == clang.cindex.CursorKind.NAMESPACE:
0154
0155
0156 if namespace == '':
0157 if child.spelling in skip_namespaces:
0158 continue
0159
0160
0161 if child.spelling.startswith('_'):
0162 continue
0163
0164 logging.debug('Going into namespace %s', child.spelling)
0165
0166 results.update(get_serializable_classes_members(child, all_template_types, namespace + child.spelling + '::', only_from_path))
0167 continue
0168
0169 if child.kind in [clang.cindex.CursorKind.CLASS_DECL, clang.cindex.CursorKind.STRUCT_DECL, clang.cindex.CursorKind.CLASS_TEMPLATE] and is_definition_by_loc(child):
0170 logging.debug('Found struct/class/template definition: %s', child.spelling if child.spelling else '<anonymous>')
0171
0172 if only_from_path is not None \
0173 and child.location.file is not None \
0174 and not child.location.file.name.startswith(only_from_path):
0175 logging.debug('Skipping since it is an external of this package: %s', child.spelling)
0176 continue
0177
0178 serializable = is_serializable_class(child)
0179 if serializable:
0180 if child.spelling == '':
0181 raise Exception('It is not possible to serialize anonymous/unnamed structs/classes.')
0182
0183 if is_serializable_class_manual(child):
0184 logging.info('Found manual serializable struct/class/template: %s', child.spelling)
0185 continue
0186
0187 logging.info('Found serializable struct/class/template: %s', child.spelling)
0188
0189 template_types = []
0190 base_objects = []
0191 members = []
0192 transients = []
0193 after_serialize = False
0194 after_serialize_count = 0
0195 for member in child.get_children():
0196 if after_serialize:
0197 if after_serialize_count == 2:
0198 after_serialize = False
0199 else:
0200 after_serialize_count = after_serialize_count + 1
0201
0202 if not is_friend_decl(member.kind):
0203 raise Exception('Expected unexposed declaration (friend) after serialize() but found something else: looks like the COND_SERIALIZABLE macro has been changed without updating the script.')
0204
0205 if 'COND_SERIALIZABLE' not in get_statement(member):
0206 raise Exception('Could not find COND_SERIALIZABLE in the statement of the expected unexposed declarations (friends) after serialize(). Please fix the script/macro.')
0207
0208 logging.debug('Skipping expected unexposed declaration (friend) after serialize().')
0209 continue
0210
0211
0212 if member.kind == clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER:
0213 logging.info(' Found template type parameter: %s', member.spelling)
0214 template_types.append(('typename', member.spelling))
0215
0216
0217 elif member.kind == clang.cindex.CursorKind.TEMPLATE_NON_TYPE_PARAMETER:
0218 type_string = get_type_string(member)
0219 if not type_string:
0220 type_string = get_basic_type_string(member)
0221 logging.info(' Found template non-type parameter: %s %s', type_string, member.spelling)
0222 template_types.append((type_string, member.spelling))
0223
0224
0225 elif member.kind == clang.cindex.CursorKind.CXX_BASE_SPECIFIER:
0226
0227 base_object = member.displayname
0228 prefix = 'class '
0229 if base_object.startswith(prefix):
0230 base_object = base_object[len(prefix):]
0231 logging.info(' Found base object: %s', base_object)
0232 base_objects.append(base_object)
0233
0234
0235 elif member.kind == clang.cindex.CursorKind.FIELD_DECL and is_definition_by_loc(member):
0236
0237
0238
0239
0240
0241
0242
0243 if 'COND_TRANSIENT' not in get_statement(member):
0244 logging.info(' Found member variable: %s', member.spelling)
0245 members.append(member.spelling)
0246 else:
0247 if serializable:
0248 logging.info(' Found transient member variable: %s', member.spelling)
0249 transients.append(member.spelling)
0250 else:
0251 raise Exception('Transient %s found for non-serializable class %s', member.spelling, child.spelling)
0252
0253 elif member.kind == clang.cindex.CursorKind.FUNCTION_TEMPLATE and member.spelling == 'serialize':
0254 after_serialize = True
0255 logging.debug('Found serialize() method, skipping next two children which must be unexposed declarations.')
0256
0257 elif member.kind in frozenset([
0258
0259
0260
0261 clang.cindex.CursorKind.CONSTRUCTOR,
0262 clang.cindex.CursorKind.DESTRUCTOR,
0263 clang.cindex.CursorKind.CXX_METHOD,
0264 clang.cindex.CursorKind.CXX_ACCESS_SPEC_DECL,
0265 clang.cindex.CursorKind.FUNCTION_TEMPLATE,
0266 clang.cindex.CursorKind.TYPEDEF_DECL,
0267 clang.cindex.CursorKind.CLASS_DECL,
0268 clang.cindex.CursorKind.ENUM_DECL,
0269 clang.cindex.CursorKind.VAR_DECL,
0270 clang.cindex.CursorKind.STRUCT_DECL,
0271 clang.cindex.CursorKind.UNION_DECL,
0272 clang.cindex.CursorKind.CONVERSION_FUNCTION,
0273 clang.cindex.CursorKind.TYPE_REF,
0274 clang.cindex.CursorKind.DECL_REF_EXPR,
0275 clang.cindex.CursorKind.CLASS_TEMPLATE,
0276 clang.cindex.CursorKind.TYPE_ALIAS_DECL,
0277 ]):
0278 logging.debug('Skipping member: %s %s %s %s', member.displayname, member.spelling, member.kind, member.type.kind)
0279
0280 elif is_friend_decl(member.kind):
0281 statement = get_statement(member)
0282
0283
0284 if 'friend' in statement:
0285
0286 if \
0287 'friend class ' in statement or \
0288 'friend struct ' in statement or \
0289 'friend std::ostream& operator<<(' in statement or \
0290 'friend std::istream& operator>>(' in statement:
0291 logging.debug('Skipping known friend: %s', statement.splitlines()[0])
0292 continue
0293
0294
0295 logging.warning('Unexposed declaration that looks like a friend declaration -- please check: %s %s %s %s %s', member.displayname, member.spelling, member.kind, member.type.kind, statement)
0296 continue
0297
0298 raise Exception('Unexposed declaration. This probably means (at the time of writing) that an unknown class was found (may happen, for instance, when the compiler does not find the headers for std::vector, i.e. missing -I option): %s %s %s %s %s' % (member.displayname, member.spelling, member.kind, member.type.kind, statement))
0299
0300 else:
0301 statement = get_statement(member)
0302 raise Exception('Unknown kind. Please fix the script: %s %s %s %s %s' % (member.displayname, member.spelling, member.kind, member.type.kind, statement))
0303
0304 if template_types:
0305 template_use = '%s<%s>' % (child.spelling, ', '.join([template_type_name for (_, template_type_name) in template_types]))
0306 else:
0307 template_use = child.spelling
0308
0309 new_namespace = namespace + template_use
0310
0311 new_all_template_types = all_template_types + [template_types]
0312
0313 results[new_namespace] = (child, serializable, new_all_template_types, base_objects, members, transients)
0314
0315 results.update(get_serializable_classes_members(child, new_all_template_types, new_namespace + '::', only_from_path))
0316
0317 for (klass, (node, serializable, all_template_types, base_objects, members, transients)) in results.items():
0318 if serializable and len(members) == 0:
0319 logging.info('No non-transient members found for serializable class %s', klass)
0320
0321 return results
0322
0323
0324 def split_path(path):
0325 folders = []
0326
0327 while True:
0328 path, folder = os.path.split(path)
0329
0330 if folder != '':
0331 folders.append(folder)
0332 else:
0333 if path != '':
0334 folders.append(path)
0335 break
0336
0337 folders.reverse()
0338
0339 return folders
0340
0341
0342 def get_flags(product_name, flags):
0343 command = "scram b echo_%s_%s | tail -1 | cut -d '=' -f '2-' | xargs -n1" % (product_name, flags)
0344 logging.debug('Running: %s', command)
0345 return subprocess.check_output(command, shell=True).splitlines()
0346
0347 def get_clang_version():
0348 """Extract clang version and set global clang_version and also return the same value."""
0349 global clang_version
0350 if clang_version is not None:
0351 return clang_version
0352 command = "clang --version | grep 'clang version' | sed 's/clang version//'"
0353 logging.debug("Running: {0}".format(command))
0354 (clang_version_major, clang_version_minor, clang_version_patchlevel) = subprocess.check_output(command, shell=True).splitlines()[0].decode('ascii').strip().split(" ")[0].split('.', 3)
0355 clang_version = (int(clang_version_major), int(clang_version_minor), int(clang_version_patchlevel))
0356 logging.debug("Detected Clang version: {0}".format(clang_version))
0357 return clang_version
0358
0359 def is_friend_decl(memkind):
0360 """Check if declaration is a friend"""
0361 clangv = get_clang_version()
0362 if clangv >= (4, 0, 0):
0363 return memkind == clang.cindex.CursorKind.FRIEND_DECL
0364 else:
0365 return memkind == clang.cindex.CursorKind.UNEXPOSED_DECL
0366 return false
0367
0368 def log_flags(name, flags):
0369 logging.debug('%s = [', name)
0370 for flag in flags:
0371 logging.debug(' %s', flag)
0372 logging.debug(']')
0373
0374
0375 def get_diagnostics(translation_unit):
0376 return map(lambda diag: {
0377 'severity' : diag.severity,
0378 'location' : diag.location,
0379 'spelling' : diag.spelling,
0380 'ranges' : diag.ranges,
0381 'fixits' : diag.fixits,
0382 }, translation_unit.diagnostics)
0383
0384
0385 def get_default_gcc_search_paths(gcc = 'g++', language = 'c++'):
0386 command = 'echo "" | %s -x%s -v -E - 2>&1' % (gcc, language)
0387 logging.debug('Running: %s', command)
0388
0389 paths = []
0390 in_list = False
0391 for line in [l.decode("ascii") for l in subprocess.check_output(command, shell=True).splitlines()]:
0392 if in_list:
0393 if line == 'End of search list.':
0394 break
0395
0396 path = os.path.normpath(line.strip())
0397
0398
0399
0400
0401 if '/lib/gcc/' in path:
0402 continue
0403
0404 paths.append('-I%s' % path)
0405
0406 else:
0407 if line == '#include <...> search starts here:':
0408 in_list = True
0409
0410 if not in_list:
0411 raise Exception('Default GCC search paths not found.')
0412
0413 return paths
0414
0415 def sanitise(var):
0416 return re.sub('[^a-zA-Z0-9.,-:]', '-', var)
0417
0418
0419 class SerializationCodeGenerator(object):
0420
0421 def __init__(self, scramFlags=None):
0422
0423 self.cmssw_base = os.getenv('CMSSW_BASE')
0424 if self.cmssw_base is None:
0425 raise Exception('CMSSW_BASE is not set.')
0426 logging.debug('cmssw_base = %s', self.cmssw_base)
0427
0428 cwd = os.getcwd()
0429 logging.debug('cwd = %s', cwd)
0430
0431 if not cwd.startswith(self.cmssw_base):
0432 raise Exception('The filepath does not start with CMSSW_BASE.')
0433
0434 relative_path = cwd[len(self.cmssw_base)+1:]
0435 logging.debug('relative_path = %s', relative_path)
0436
0437 self.split_path = split_path(relative_path)
0438 logging.debug('splitpath = %s', self.split_path)
0439
0440 if len(self.split_path) < 3:
0441 raise Exception('This script requires to be run inside a CMSSW package (usually within CondFormats), e.g. CondFormats/Alignment. The current path is: %s' % self.split_path)
0442
0443 if self.split_path[0] != 'src':
0444 raise Exception('The first folder should be src.')
0445
0446 if self.split_path[1] != 'CondFormats':
0447 raise Exception('The second folder should be CondFormats.')
0448
0449 product_name = '%s%s' % (self.split_path[1], self.split_path[2])
0450 logging.debug('product_name = %s', product_name)
0451
0452 if not scramFlags:
0453 cpp_flags = get_flags(product_name, 'CPPFLAGS')
0454 cxx_flags = get_flags(product_name, 'CXXFLAGS')
0455 else:
0456 cpp_flags = self.cleanFlags( scramFlags )
0457 cxx_flags = []
0458
0459
0460 std_flags = get_default_gcc_search_paths(gcc='clang++')
0461 log_flags('cpp_flags', cpp_flags)
0462 log_flags('cxx_flags', cxx_flags)
0463 log_flags('std_flags', std_flags)
0464
0465 flags = ['-xc++'] + cpp_flags + cxx_flags + std_flags
0466
0467 headers_h = self._join_package_path('src', 'headers.h')
0468 logging.debug('headers_h = %s', headers_h)
0469 if not os.path.exists(headers_h):
0470 raise Exception('File %s does not exist. Impossible to serialize package.' % headers_h)
0471
0472 logging.info('Searching serializable classes in %s/%s ...', self.split_path[1], self.split_path[2])
0473
0474 logging.debug('Parsing C++ classes in file %s ...', headers_h)
0475
0476 if "SCRAM_ARCH" in os.environ and re.match('osx10*',os.environ['SCRAM_ARCH']):
0477 cindex=clang.cindex
0478 libpath=os.path.dirname(os.path.realpath(clang.cindex.__file__))+"/../../lib"
0479 cindex.Config.set_library_path(libpath)
0480 index = cindex.Index.create()
0481 else :
0482 index = clang.cindex.Index.create()
0483 translation_unit = index.parse(headers_h, flags)
0484 if not translation_unit:
0485 raise Exception('Unable to load input.')
0486
0487 severity_names = ('Ignored', 'Note', 'Warning', 'Error', 'Fatal')
0488 get_severity_name = lambda severity_num: severity_names[severity_num] if severity_num < len(severity_names) else 'Unknown'
0489 max_severity_level = 0
0490 diagnostics = get_diagnostics(translation_unit)
0491 for diagnostic in diagnostics:
0492 logf = logging.error
0493
0494
0495 if diagnostic['spelling'].startswith('argument unused during compilation') \
0496 or diagnostic['spelling'].startswith('unknown warning option'):
0497 logf = logging.debug
0498
0499 logf('Diagnostic: [%s] %s', get_severity_name(diagnostic['severity']), diagnostic['spelling'])
0500 logf(' at line %s in %s', diagnostic['location'].line, diagnostic['location'].file)
0501
0502 max_severity_level = max(max_severity_level, diagnostic['severity'])
0503
0504 if max_severity_level >= 3:
0505 raise Exception('Please, resolve all errors before proceeding.')
0506
0507 self.classes = get_serializable_classes_members(translation_unit.cursor, only_from_path=self._join_package_path())
0508
0509 def _join_package_path(self, *path):
0510 return os.path.join(self.cmssw_base, self.split_path[0], self.split_path[1], self.split_path[2], *path)
0511
0512 def cleanFlags(self, flagsIn):
0513 flags = [ flag for flag in flagsIn if not flag.startswith(('-march', '-mtune', '-fdebug-prefix-map', '-ax', '-wd', '-fsanitize=')) ]
0514 blackList = ['--', '-fipa-pta', '-xSSE3', '-fno-crossjumping', '-fno-aggressive-loop-optimizations']
0515 return [x for x in flags if x not in blackList]
0516
0517 def generate(self, outFileName):
0518
0519 filename = outFileName
0520 if not filename:
0521 filename = self._join_package_path('src', 'Serialization.cc')
0522
0523 n_serializable_classes = 0
0524
0525 source = headers_template.format(headers=os.path.join(self.split_path[1], self.split_path[2], 'src', 'headers.h'))
0526
0527 for klass in sorted(self.classes):
0528 (node, serializable, all_template_types, base_objects, members, transients) = self.classes[klass]
0529
0530 if not serializable:
0531 continue
0532
0533 n_serializable_classes += 1
0534
0535 skip_instantiation = False
0536 for template_types in all_template_types:
0537 if template_types:
0538 skip_instantiation = True
0539 source += ('template <%s>' % ', '.join(['%s %s' % template_type for template_type in template_types])) + '\n'
0540
0541 source += serialize_method_begin_template.format(klass=klass) + '\n'
0542
0543 for base_object_name in base_objects:
0544 base_object_name_sanitised = sanitise(base_object_name)
0545 source += serialize_method_base_object_template.format(base_object_name=base_object_name, base_object_name_sanitised=base_object_name_sanitised) + '\n'
0546
0547 for member_name in members:
0548 member_name_sanitised = sanitise(member_name)
0549 source += serialize_method_member_template.format(member_name=member_name, member_name_sanitised=member_name_sanitised) + '\n'
0550
0551 source += serialize_method_end
0552
0553 if skip_instantiation:
0554 source += '\n'
0555 else:
0556 source += instantiation_template.format(klass=klass) + '\n'
0557
0558 if n_serializable_classes == 0:
0559 raise Exception('No serializable classes found, while this package has a headers.h file.')
0560
0561
0562 if os.path.exists( './src/SerializationManual.h' ) :
0563 source += '#include "%s/%s/src/SerializationManual.h"\n' % (self.split_path[1], self.split_path[2])
0564
0565 logging.info('Writing serialization code for %s classes in %s ...', n_serializable_classes, filename)
0566 with open(filename, 'w') as fd:
0567 fd.write(source)
0568
0569
0570 def main():
0571 parser = argparse.ArgumentParser(description='CMS Condition DB Serialization generator.')
0572 parser.add_argument('--verbose', '-v', action='count', help='Verbosity level. -v reports debugging information.', default=0)
0573 parser.add_argument('--output' , '-o', action='store', help='Specifies the path to the output file written. Default: src/Serialization.cc')
0574 parser.add_argument('--package', '-p', action='store', help='Specifies the path to the package to be processed. Default: the actual package')
0575
0576 opts, args = parser.parse_known_args()
0577
0578 logLevel = logging.INFO
0579 if opts.verbose < 1 and opts.output and opts.package:
0580 logLevel = logging.WARNING
0581
0582 if opts.verbose >= 1:
0583 logLevel = logging.DEBUG
0584
0585 logging.basicConfig(
0586 format = '[%(asctime)s] %(levelname)s: %(message)s',
0587 level = logLevel,
0588 )
0589
0590 if opts.package:
0591 pkgDir = opts.package
0592 if pkgDir.endswith('/src') :
0593 pkgDir, srcDir = os.path.split( opts.package )
0594 os.chdir( pkgDir )
0595 logging.info("Processing package in %s " % pkgDir)
0596
0597 if opts.output:
0598 logging.info("Writing serialization code to %s " % opts.output)
0599
0600 SerializationCodeGenerator( scramFlags=args[1:] ).generate( opts.output )
0601
0602 if __name__ == '__main__':
0603 main()
0604