File indexing completed on 2024-04-06 12:02:33
0001
0002 from __future__ import print_function
0003 '''CMS Conditions DB Serialization generator.
0004
0005 Generates the non-intrusive serialization code required for the classes
0006 marked with the COND_SERIALIZABLE macro.
0007
0008 The code was taken from the prototype that did many other things as well
0009 (finding transients, marking serializable classes, etc.). After removing
0010 everything but what is required to build the serialization, the code was
0011 made more robust and cleaned up a bit to be integrated on the BoostIO IB.
0012 However, the code still needs to be restructured a bit more to improve
0013 readability (e.g. name some constants, use a template engine, ask for
0014 clang's bindings to be installed along clang itself, etc.).
0015 '''
0016
0017 __author__ = 'Miguel Ojeda'
0018 __copyright__ = 'Copyright 2014, CERN'
0019 __credits__ = ['Giacomo Govi', 'Miguel Ojeda', 'Andreas Pfeiffer']
0020 __license__ = 'Unknown'
0021 __maintainer__ = 'Miguel Ojeda'
0022 __email__ = 'mojedasa@cern.ch'
0023
0024
0025 import argparse
0026 import logging
0027 import os
0028 import re
0029 import subprocess
0030
0031 import clang.cindex
0032
0033 clang_version = None
0034
0035 headers_template = '''
0036 #include "{headers}"
0037
0038 #include <boost/serialization/base_object.hpp>
0039 #include <boost/serialization/nvp.hpp>
0040 #include <boost/serialization/export.hpp>
0041
0042 #include "CondFormats/Serialization/interface/Equal.h"
0043 #include "CondFormats/Serialization/interface/Instantiate.h"
0044
0045 '''
0046
0047 serialize_method_begin_template = '''template <class Archive>
0048 void {klass}::serialize(Archive & ar, const unsigned int)
0049 {{'''
0050
0051 serialize_method_base_object_template = ' ar & boost::serialization::make_nvp("{base_object_name_sanitised}", boost::serialization::base_object<{base_object_name}>(*this));'
0052
0053 serialize_method_member_template = ''' ar & boost::serialization::make_nvp("{member_name_sanitised}", {member_name});'''
0054
0055 serialize_method_end = '''}
0056 '''
0057
0058 instantiation_template = '''COND_SERIALIZATION_INSTANTIATE({klass});
0059 '''
0060
0061
0062 skip_namespaces = frozenset([
0063
0064 '',
0065
0066
0067 'std', 'boost', 'mpl_', 'boost_swap_impl',
0068
0069
0070 'ROOT', 'edm', 'ora', 'coral', 'CLHEP', 'Geom', 'HepGeom',
0071 ])
0072
0073 def is_definition_by_loc(node):
0074 if node.get_definition() is None:
0075 return False
0076 if node.location is None or node.get_definition().location is None:
0077 return False
0078 return node.location == node.get_definition().location
0079
0080 def is_serializable_class(node):
0081 for child in node.get_children():
0082 if child.spelling != 'serialize' or child.kind != clang.cindex.CursorKind.FUNCTION_TEMPLATE or is_definition_by_loc(child):
0083 continue
0084
0085 if [(x.spelling, x.kind, is_definition_by_loc(x), x.type.kind) for x in child.get_children()] != [
0086 ('Archive', clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER, True, clang.cindex.TypeKind.UNEXPOSED),
0087 ('ar', clang.cindex.CursorKind.PARM_DECL, True, clang.cindex.TypeKind.LVALUEREFERENCE),
0088 ('version', clang.cindex.CursorKind.PARM_DECL, True, clang.cindex.TypeKind.UINT),
0089 ]:
0090 continue
0091
0092 return True
0093
0094 return False
0095
0096
0097 def is_serializable_class_manual(node):
0098 for child in node.get_children():
0099 if child.spelling == 'cond_serialization_manual' and child.kind == clang.cindex.CursorKind.CXX_METHOD and not is_definition_by_loc(child):
0100 return True
0101
0102 return False
0103
0104
0105 def get_statement(node):
0106
0107
0108
0109 if node.extent.start.file is None:
0110 return None
0111
0112 filename = node.extent.start.file.name
0113 start = node.extent.start.offset
0114 end = node.extent.end.offset
0115
0116 with open(filename, 'rb') as fd:
0117 source = fd.read().decode('latin-1')
0118
0119 return source[start:source.find(';', end)]
0120
0121
0122 def get_basic_type_string(node):
0123 typekinds = {
0124 clang.cindex.TypeKind.BOOL: 'bool',
0125 clang.cindex.TypeKind.INT: 'int',
0126 clang.cindex.TypeKind.LONG: 'long',
0127 clang.cindex.TypeKind.UINT: 'unsigned int',
0128 clang.cindex.TypeKind.ULONG: 'unsigned long',
0129 clang.cindex.TypeKind.FLOAT: 'float',
0130 clang.cindex.TypeKind.DOUBLE: 'double',
0131 }
0132
0133 if node.type.kind not in typekinds:
0134 raise Exception('Not a known basic type.')
0135
0136 return typekinds[node.type.kind]
0137
0138
0139 def get_type_string(node):
0140 spelling = node.type.get_declaration().spelling
0141 if spelling is not None:
0142 return spelling
0143
0144 return get_basic_type_string(node)
0145
0146
0147 def get_serializable_classes_members(node, all_template_types=None, namespace='', only_from_path=None):
0148 if all_template_types is None:
0149 all_template_types = []
0150
0151 logging.debug('%s', (node.spelling, all_template_types, namespace))
0152 results = {}
0153 for child in node.get_children():
0154 if child.kind == clang.cindex.CursorKind.NAMESPACE:
0155
0156
0157 if namespace == '':
0158 if child.spelling in skip_namespaces:
0159 continue
0160
0161
0162 if child.spelling.startswith('_'):
0163 continue
0164
0165 logging.debug('Going into namespace %s', child.spelling)
0166
0167 results.update(get_serializable_classes_members(child, all_template_types, namespace + child.spelling + '::', only_from_path))
0168 continue
0169
0170 if child.kind in [clang.cindex.CursorKind.CLASS_DECL, clang.cindex.CursorKind.STRUCT_DECL, clang.cindex.CursorKind.CLASS_TEMPLATE] and is_definition_by_loc(child):
0171 logging.debug('Found struct/class/template definition: %s', child.spelling if child.spelling else '<anonymous>')
0172
0173 if only_from_path is not None \
0174 and child.location.file is not None \
0175 and not child.location.file.name.startswith(only_from_path):
0176 logging.debug('Skipping since it is an external of this package: %s', child.spelling)
0177 continue
0178
0179 serializable = is_serializable_class(child)
0180 if serializable:
0181 if child.spelling == '':
0182 raise Exception('It is not possible to serialize anonymous/unnamed structs/classes.')
0183
0184 if is_serializable_class_manual(child):
0185 logging.info('Found manual serializable struct/class/template: %s', child.spelling)
0186 continue
0187
0188 logging.info('Found serializable struct/class/template: %s', child.spelling)
0189
0190 template_types = []
0191 base_objects = []
0192 members = []
0193 transients = []
0194 after_serialize = False
0195 after_serialize_count = 0
0196 for member in child.get_children():
0197 if after_serialize:
0198 if after_serialize_count == 2:
0199 after_serialize = False
0200 else:
0201 after_serialize_count = after_serialize_count + 1
0202
0203 if not is_friend_decl(member.kind):
0204 raise Exception('Expected unexposed declaration (friend) after serialize() but found something else: looks like the COND_SERIALIZABLE macro has been changed without updating the script.')
0205
0206 if 'COND_SERIALIZABLE' not in get_statement(member):
0207 raise Exception('Could not find COND_SERIALIZABLE in the statement of the expected unexposed declarations (friends) after serialize(). Please fix the script/macro.')
0208
0209 logging.debug('Skipping expected unexposed declaration (friend) after serialize().')
0210 continue
0211
0212
0213 if member.kind == clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER:
0214 logging.info(' Found template type parameter: %s', member.spelling)
0215 template_types.append(('typename', member.spelling))
0216
0217
0218 elif member.kind == clang.cindex.CursorKind.TEMPLATE_NON_TYPE_PARAMETER:
0219 type_string = get_type_string(member)
0220 if not type_string:
0221 type_string = get_basic_type_string(member)
0222 logging.info(' Found template non-type parameter: %s %s', type_string, member.spelling)
0223 template_types.append((type_string, member.spelling))
0224
0225
0226 elif member.kind == clang.cindex.CursorKind.CXX_BASE_SPECIFIER:
0227
0228 base_object = member.displayname
0229 prefix = 'class '
0230 if base_object.startswith(prefix):
0231 base_object = base_object[len(prefix):]
0232 logging.info(' Found base object: %s', base_object)
0233 base_objects.append(base_object)
0234
0235
0236 elif member.kind == clang.cindex.CursorKind.FIELD_DECL and is_definition_by_loc(member):
0237
0238
0239
0240
0241
0242
0243
0244 if 'COND_TRANSIENT' not in get_statement(member):
0245 logging.info(' Found member variable: %s', member.spelling)
0246 members.append(member.spelling)
0247 else:
0248 if serializable:
0249 logging.info(' Found transient member variable: %s', member.spelling)
0250 transients.append(member.spelling)
0251 else:
0252 raise Exception('Transient %s found for non-serializable class %s', member.spelling, child.spelling)
0253
0254 elif member.kind == clang.cindex.CursorKind.FUNCTION_TEMPLATE and member.spelling == 'serialize':
0255 after_serialize = True
0256 logging.debug('Found serialize() method, skipping next two children which must be unexposed declarations.')
0257
0258 elif member.kind in frozenset([
0259
0260
0261
0262 clang.cindex.CursorKind.CONSTRUCTOR,
0263 clang.cindex.CursorKind.DESTRUCTOR,
0264 clang.cindex.CursorKind.CXX_METHOD,
0265 clang.cindex.CursorKind.CXX_ACCESS_SPEC_DECL,
0266 clang.cindex.CursorKind.FUNCTION_TEMPLATE,
0267 clang.cindex.CursorKind.TYPEDEF_DECL,
0268 clang.cindex.CursorKind.CLASS_DECL,
0269 clang.cindex.CursorKind.ENUM_DECL,
0270 clang.cindex.CursorKind.VAR_DECL,
0271 clang.cindex.CursorKind.STRUCT_DECL,
0272 clang.cindex.CursorKind.UNION_DECL,
0273 clang.cindex.CursorKind.CONVERSION_FUNCTION,
0274 clang.cindex.CursorKind.TYPE_REF,
0275 clang.cindex.CursorKind.DECL_REF_EXPR,
0276 clang.cindex.CursorKind.CLASS_TEMPLATE,
0277 clang.cindex.CursorKind.TYPE_ALIAS_DECL,
0278 ]):
0279 logging.debug('Skipping member: %s %s %s %s', member.displayname, member.spelling, member.kind, member.type.kind)
0280
0281 elif is_friend_decl(member.kind):
0282 statement = get_statement(member)
0283
0284
0285 if 'friend' in statement:
0286
0287 if \
0288 'friend class ' in statement or \
0289 'friend struct ' in statement or \
0290 'friend std::ostream& operator<<(' in statement or \
0291 'friend std::istream& operator>>(' in statement:
0292 logging.debug('Skipping known friend: %s', statement.splitlines()[0])
0293 continue
0294
0295
0296 logging.warning('Unexposed declaration that looks like a friend declaration -- please check: %s %s %s %s %s', member.displayname, member.spelling, member.kind, member.type.kind, statement)
0297 continue
0298
0299 raise Exception('Unexposed declaration. This probably means (at the time of writing) that an unknown class was found (may happen, for instance, when the compiler does not find the headers for std::vector, i.e. missing -I option): %s %s %s %s %s' % (member.displayname, member.spelling, member.kind, member.type.kind, statement))
0300
0301 else:
0302 statement = get_statement(member)
0303 raise Exception('Unknown kind. Please fix the script: %s %s %s %s %s' % (member.displayname, member.spelling, member.kind, member.type.kind, statement))
0304
0305 if template_types:
0306 template_use = '%s<%s>' % (child.spelling, ', '.join([template_type_name for (_, template_type_name) in template_types]))
0307 else:
0308 template_use = child.spelling
0309
0310 new_namespace = namespace + template_use
0311
0312 new_all_template_types = all_template_types + [template_types]
0313
0314 results[new_namespace] = (child, serializable, new_all_template_types, base_objects, members, transients)
0315
0316 results.update(get_serializable_classes_members(child, new_all_template_types, new_namespace + '::', only_from_path))
0317
0318 for (klass, (node, serializable, all_template_types, base_objects, members, transients)) in results.items():
0319 if serializable and len(members) == 0:
0320 logging.info('No non-transient members found for serializable class %s', klass)
0321
0322 return results
0323
0324
0325 def split_path(path):
0326 folders = []
0327
0328 while True:
0329 path, folder = os.path.split(path)
0330
0331 if folder != '':
0332 folders.append(folder)
0333 else:
0334 if path != '':
0335 folders.append(path)
0336 break
0337
0338 folders.reverse()
0339
0340 return folders
0341
0342
0343 def get_flags(product_name, flags):
0344 command = "scram b echo_%s_%s | tail -1 | cut -d '=' -f '2-' | xargs -n1" % (product_name, flags)
0345 logging.debug('Running: %s', command)
0346 return subprocess.check_output(command, shell=True).splitlines()
0347
0348 def get_clang_version():
0349 """Extract clang version and set global clang_version and also return the same value."""
0350 global clang_version
0351 if clang_version is not None:
0352 return clang_version
0353 command = "clang --version | grep 'clang version' | sed 's/clang version//'"
0354 logging.debug("Running: {0}".format(command))
0355 (clang_version_major, clang_version_minor, clang_version_patchlevel) = subprocess.check_output(command, shell=True).splitlines()[0].decode('ascii').strip().split(" ")[0].split('.', 3)
0356 clang_version = (int(clang_version_major), int(clang_version_minor), int(clang_version_patchlevel))
0357 logging.debug("Detected Clang version: {0}".format(clang_version))
0358 return clang_version
0359
0360 def is_friend_decl(memkind):
0361 """Check if declaration is a friend"""
0362 clangv = get_clang_version()
0363 if clangv >= (4, 0, 0):
0364 return memkind == clang.cindex.CursorKind.FRIEND_DECL
0365 else:
0366 return memkind == clang.cindex.CursorKind.UNEXPOSED_DECL
0367 return false
0368
0369 def log_flags(name, flags):
0370 logging.debug('%s = [', name)
0371 for flag in flags:
0372 logging.debug(' %s', flag)
0373 logging.debug(']')
0374
0375
0376 def get_diagnostics(translation_unit):
0377 return map(lambda diag: {
0378 'severity' : diag.severity,
0379 'location' : diag.location,
0380 'spelling' : diag.spelling,
0381 'ranges' : diag.ranges,
0382 'fixits' : diag.fixits,
0383 }, translation_unit.diagnostics)
0384
0385
0386 def get_default_gcc_search_paths(gcc = 'g++', language = 'c++'):
0387 command = 'echo "" | %s -x%s -v -E - 2>&1' % (gcc, language)
0388 logging.debug('Running: %s', command)
0389
0390 paths = []
0391 in_list = False
0392 for line in [l.decode("ascii") for l in subprocess.check_output(command, shell=True).splitlines()]:
0393 if in_list:
0394 if line == 'End of search list.':
0395 break
0396
0397 path = os.path.normpath(line.strip())
0398
0399
0400
0401
0402 if '/lib/gcc/' in path:
0403 continue
0404
0405 paths.append('-I%s' % path)
0406
0407 else:
0408 if line == '#include <...> search starts here:':
0409 in_list = True
0410
0411 if not in_list:
0412 raise Exception('Default GCC search paths not found.')
0413
0414 return paths
0415
0416 def sanitise(var):
0417 return re.sub('[^a-zA-Z0-9.,-:]', '-', var)
0418
0419
0420 class SerializationCodeGenerator(object):
0421
0422 def __init__(self, scramFlags=None):
0423
0424 self.cmssw_base = os.getenv('CMSSW_BASE')
0425 if self.cmssw_base is None:
0426 raise Exception('CMSSW_BASE is not set.')
0427 logging.debug('cmssw_base = %s', self.cmssw_base)
0428
0429 cwd = os.getcwd()
0430 logging.debug('cwd = %s', cwd)
0431
0432 if not cwd.startswith(self.cmssw_base):
0433 raise Exception('The filepath does not start with CMSSW_BASE.')
0434
0435 relative_path = cwd[len(self.cmssw_base)+1:]
0436 logging.debug('relative_path = %s', relative_path)
0437
0438 self.split_path = split_path(relative_path)
0439 logging.debug('splitpath = %s', self.split_path)
0440
0441 if len(self.split_path) < 3:
0442 raise Exception('This script requires to be run inside a CMSSW package (usually within CondFormats), e.g. CondFormats/Alignment. The current path is: %s' % self.split_path)
0443
0444 if self.split_path[0] != 'src':
0445 raise Exception('The first folder should be src.')
0446
0447 if self.split_path[1] != 'CondFormats':
0448 raise Exception('The second folder should be CondFormats.')
0449
0450 product_name = '%s%s' % (self.split_path[1], self.split_path[2])
0451 logging.debug('product_name = %s', product_name)
0452
0453 if not scramFlags:
0454 cpp_flags = get_flags(product_name, 'CPPFLAGS')
0455 cxx_flags = get_flags(product_name, 'CXXFLAGS')
0456 else:
0457 cpp_flags = self.cleanFlags( scramFlags )
0458 cxx_flags = []
0459
0460
0461 std_flags = get_default_gcc_search_paths(gcc='clang++')
0462 log_flags('cpp_flags', cpp_flags)
0463 log_flags('cxx_flags', cxx_flags)
0464 log_flags('std_flags', std_flags)
0465
0466 flags = ['-xc++'] + cpp_flags + cxx_flags + std_flags
0467
0468 headers_h = self._join_package_path('src', 'headers.h')
0469 logging.debug('headers_h = %s', headers_h)
0470 if not os.path.exists(headers_h):
0471 raise Exception('File %s does not exist. Impossible to serialize package.' % headers_h)
0472
0473 logging.info('Searching serializable classes in %s/%s ...', self.split_path[1], self.split_path[2])
0474
0475 logging.debug('Parsing C++ classes in file %s ...', headers_h)
0476
0477 if "SCRAM_ARCH" in os.environ and re.match('osx10*',os.environ['SCRAM_ARCH']):
0478 cindex=clang.cindex
0479 libpath=os.path.dirname(os.path.realpath(clang.cindex.__file__))+"/../../lib"
0480 cindex.Config.set_library_path(libpath)
0481 index = cindex.Index.create()
0482 else :
0483 index = clang.cindex.Index.create()
0484 translation_unit = index.parse(headers_h, flags)
0485 if not translation_unit:
0486 raise Exception('Unable to load input.')
0487
0488 severity_names = ('Ignored', 'Note', 'Warning', 'Error', 'Fatal')
0489 get_severity_name = lambda severity_num: severity_names[severity_num] if severity_num < len(severity_names) else 'Unknown'
0490 max_severity_level = 0
0491 diagnostics = get_diagnostics(translation_unit)
0492 for diagnostic in diagnostics:
0493 logf = logging.error
0494
0495
0496 if diagnostic['spelling'].startswith('argument unused during compilation') \
0497 or diagnostic['spelling'].startswith('unknown warning option'):
0498 logf = logging.debug
0499
0500 logf('Diagnostic: [%s] %s', get_severity_name(diagnostic['severity']), diagnostic['spelling'])
0501 logf(' at line %s in %s', diagnostic['location'].line, diagnostic['location'].file)
0502
0503 max_severity_level = max(max_severity_level, diagnostic['severity'])
0504
0505 if max_severity_level >= 3:
0506 raise Exception('Please, resolve all errors before proceeding.')
0507
0508 self.classes = get_serializable_classes_members(translation_unit.cursor, only_from_path=self._join_package_path())
0509
0510 def _join_package_path(self, *path):
0511 return os.path.join(self.cmssw_base, self.split_path[0], self.split_path[1], self.split_path[2], *path)
0512
0513 def cleanFlags(self, flagsIn):
0514 flags = [ flag for flag in flagsIn if not flag.startswith(('-march', '-mtune', '-fdebug-prefix-map', '-ax', '-wd', '-fsanitize=')) ]
0515 blackList = ['--', '-fipa-pta', '-xSSE3', '-fno-crossjumping', '-fno-aggressive-loop-optimizations']
0516 return [x for x in flags if x not in blackList]
0517
0518 def generate(self, outFileName):
0519
0520 filename = outFileName
0521 if not filename:
0522 filename = self._join_package_path('src', 'Serialization.cc')
0523
0524 n_serializable_classes = 0
0525
0526 source = headers_template.format(headers=os.path.join(self.split_path[1], self.split_path[2], 'src', 'headers.h'))
0527
0528 for klass in sorted(self.classes):
0529 (node, serializable, all_template_types, base_objects, members, transients) = self.classes[klass]
0530
0531 if not serializable:
0532 continue
0533
0534 n_serializable_classes += 1
0535
0536 skip_instantiation = False
0537 for template_types in all_template_types:
0538 if template_types:
0539 skip_instantiation = True
0540 source += ('template <%s>' % ', '.join(['%s %s' % template_type for template_type in template_types])) + '\n'
0541
0542 source += serialize_method_begin_template.format(klass=klass) + '\n'
0543
0544 for base_object_name in base_objects:
0545 base_object_name_sanitised = sanitise(base_object_name)
0546 source += serialize_method_base_object_template.format(base_object_name=base_object_name, base_object_name_sanitised=base_object_name_sanitised) + '\n'
0547
0548 for member_name in members:
0549 member_name_sanitised = sanitise(member_name)
0550 source += serialize_method_member_template.format(member_name=member_name, member_name_sanitised=member_name_sanitised) + '\n'
0551
0552 source += serialize_method_end
0553
0554 if skip_instantiation:
0555 source += '\n'
0556 else:
0557 source += instantiation_template.format(klass=klass) + '\n'
0558
0559 if n_serializable_classes == 0:
0560 raise Exception('No serializable classes found, while this package has a headers.h file.')
0561
0562
0563 if os.path.exists( './src/SerializationManual.h' ) :
0564 source += '#include "%s/%s/src/SerializationManual.h"\n' % (self.split_path[1], self.split_path[2])
0565
0566 logging.info('Writing serialization code for %s classes in %s ...', n_serializable_classes, filename)
0567 with open(filename, 'w') as fd:
0568 fd.write(source)
0569
0570
0571 def main():
0572 parser = argparse.ArgumentParser(description='CMS Condition DB Serialization generator.')
0573 parser.add_argument('--verbose', '-v', action='count', help='Verbosity level. -v reports debugging information.', default=0)
0574 parser.add_argument('--output' , '-o', action='store', help='Specifies the path to the output file written. Default: src/Serialization.cc')
0575 parser.add_argument('--package', '-p', action='store', help='Specifies the path to the package to be processed. Default: the actual package')
0576
0577 opts, args = parser.parse_known_args()
0578
0579 logLevel = logging.INFO
0580 if opts.verbose < 1 and opts.output and opts.package:
0581 logLevel = logging.WARNING
0582
0583 if opts.verbose >= 1:
0584 logLevel = logging.DEBUG
0585
0586 logging.basicConfig(
0587 format = '[%(asctime)s] %(levelname)s: %(message)s',
0588 level = logLevel,
0589 )
0590
0591 if opts.package:
0592 pkgDir = opts.package
0593 if pkgDir.endswith('/src') :
0594 pkgDir, srcDir = os.path.split( opts.package )
0595 os.chdir( pkgDir )
0596 logging.info("Processing package in %s " % pkgDir)
0597
0598 if opts.output:
0599 logging.info("Writing serialization code to %s " % opts.output)
0600
0601 SerializationCodeGenerator( scramFlags=args[1:] ).generate( opts.output )
0602
0603 if __name__ == '__main__':
0604 main()
0605