Utilities/python/conddb_migrate.py

0001 #!/usr/bin/env python3
0002 '''CMS Conditions DB migration script.
0003 '''
0004
0005
0006 import os
0007 import sys
0008 import logging
0009 import argparse
0010 import subprocess
0011 import time
0012 import multiprocessing
0013
0014 import cx_Oracle
0015
0016
0017 accounts = [
0018     #'CMS_COND_TEMP', # not in payloadInspector
0019     'CMS_COND_31X_ALIGNMENT',
0020     'CMS_COND_31X_BEAMSPOT',
0021     'CMS_COND_31X_BTAU',
0022     'CMS_COND_31X_CSC',
0023     'CMS_COND_31X_DQM_SUMMARY',
0024     'CMS_COND_31X_DT',
0025     'CMS_COND_31X_ECAL',
0026     'CMS_COND_31X_FROM21X',
0027     'CMS_COND_31X_GEOMETRY',
0028     'CMS_COND_31X_HCAL',
0029     'CMS_COND_31X_HLT',
0030     'CMS_COND_31X_L1T',
0031     'CMS_COND_31X_PHYSICSTOOLS',
0032     'CMS_COND_31X_PIXEL',
0033     'CMS_COND_31X_PRESHOWER',
0034     'CMS_COND_31X_RPC',
0035     'CMS_COND_31X_RUN_INFO',
0036     'CMS_COND_31X_STRIP',
0037     'CMS_COND_34X_DQM',
0038     'CMS_COND_34X_ECAL',
0039     'CMS_COND_34X_ECAL_PED',
0040     'CMS_COND_34X_GEOMETRY',
0041     'CMS_COND_36X_RPC',
0042     'CMS_COND_38X_HCAL',
0043     #'CMS_COND_38X_PIXEL', # not in payloadInspector
0044     'CMS_COND_39X_PRESHOWER',
0045     #'CMS_COND_310X_ALIGN', # FIXME: Segmentation fault
0046     'CMS_COND_310X_CSC',
0047     'CMS_COND_310X_ECAL_PED',
0048     'CMS_COND_311X_ECAL_LAS',
0049     'CMS_COND_311X_PRESH',
0050     'CMS_COND_42X_DQM',
0051     'CMS_COND_42X_ECAL_LAS',
0052     'CMS_COND_42X_ECAL_LASP',
0053     'CMS_COND_42X_GEOMETRY',
0054     'CMS_COND_42X_HCAL',
0055     'CMS_COND_42X_RUN_INFO',
0056     'CMS_COND_43X_ECAL',
0057     #'CMS_COND_43X_RPC_NOISE', # not in payloadInspector
0058     'CMS_COND_44X_ECAL',
0059     'CMS_COND_44X_GEOMETRY',
0060     'CMS_COND_44X_HCAL',
0061     'CMS_COND_44X_PHYSICSTOOLS',
0062     #'CMS_COND_44X_RPC', # not in payloadInspector
0063     'CMS_COND_ALIGN_000',
0064     'CMS_COND_BEAMSPOT_000',
0065     'CMS_COND_BTAU_000',
0066     'CMS_COND_CSC_000',
0067     'CMS_COND_DQM_000',
0068     'CMS_COND_DT_000',
0069     'CMS_COND_ECAL_000',
0070     'CMS_COND_ECAL_LAS_000',
0071     'CMS_COND_ECAL_PED_000',
0072     'CMS_COND_GEOMETRY_000',
0073     'CMS_COND_HCAL_000',
0074     'CMS_COND_HLT_000',
0075     'CMS_COND_L1T_000',
0076     'CMS_COND_MC_000',
0077     'CMS_COND_PAT_000',
0078     'CMS_COND_PIXEL_000',
0079     'CMS_COND_PRESH_000',
0080     'CMS_COND_RPC_000',
0081     'CMS_COND_RUNINFO_000',
0082     'CMS_COND_STRIP_000',
0083 ]
0084
0085
0086 def run_command(command, output_file):
0087     command = '%s > %s 2>&1' % (command, output_file)
0088     logging.info('Running %s', command)
0089     try:
0090         subprocess.check_call(command, shell=True)
0091     except subprocess.CalledProcessError as e:
0092         logging.error('Error while running %s: return code %s', command, e.returncode)
0093
0094
0095 def migrate_account(args):
0096     command_template = '$CMSSW_BASE/bin/$SCRAM_ARCH/conddb_migrate -s oracle://cms_orcon_adg/%s -d %s'
0097     command = command_template % (args.account, args.db)
0098     run_command(command, os.path.join(args.output, args.account))
0099
0100
0101 def migrate_accounts(args):
0102     def _make_args(args, account):
0103         newargs = argparse.Namespace(**vars(args))
0104         newargs.account = account
0105         return newargs
0106
0107     multiprocessing.Pool(args.jobs).map(migrate_account, [_make_args(args, account) for account in accounts])
0108
0109
0110 def migrate_gt(args):
0111     command_template = '$CMSSW_BASE/bin/$SCRAM_ARCH/conddb_migrate_gt -s oracle://cms_orcon_adg/CMS_COND_31X_GLOBALTAG -d %s -g %s'
0112     command = command_template % (args.db, args.gt)
0113     run_command(command, os.path.join(args.output, args.gt))
0114
0115
0116 def make_gt_connection(args):
0117     logging.info('Fetching global tag list...')
0118     password = subprocess.check_output('''cat %s | grep -F 'CMS_COND_31X_GLOBALTAG' -2 | tail -1 | cut -d'"' -f4''' % os.path.join(args.authpath, 'readOnlyProd.xml'), shell=True).strip()
0119     return cx_Oracle.connect('CMS_COND_GENERAL_R', password, 'cms_orcon_adg')
0120
0121
0122 def fetch_gts(connection):
0123     logging.info('Fetching global tag list...')
0124     cursor = connection.cursor()
0125     cursor.execute('''
0126         select substr(table_name, length('tagtree_table_') + 1) gt
0127         from all_tables
0128         where owner = 'CMS_COND_31X_GLOBALTAG'
0129             and table_name like 'TAGTREE_TABLE_%'
0130         order by gt
0131     ''')
0132     gts = zip(*cursor.fetchall())[0]
0133     logging.info('Fetching global tag list... Done: %s global tags found.', len(gts))
0134     return gts
0135
0136
0137 def migrate_gts(args):
0138     gts = fetch_gts(make_gt_connection(args))
0139
0140     def _make_args(args, gt):
0141         newargs = argparse.Namespace(**vars(args))
0142         newargs.gt = gt
0143         return newargs
0144
0145     multiprocessing.Pool(args.jobs).map(migrate_gt, [_make_args(args, gt) for gt in gts])
0146
0147
0148 def tags_in_gts(args):
0149     # Dynamic SQL is used due to the schema
0150     # This is OK since we trust the input,
0151     # which is the GT database's tables.
0152
0153     connection = make_gt_connection(args)
0154     gts = fetch_gts(connection)
0155     account_tags = {}
0156
0157     for i, gt in enumerate(gts):
0158         logging.info('[%s/%s] Reading %s ...', i+1, len(gts), gt)
0159         cursor = connection.cursor()
0160         cursor.execute('''
0161             select "pfn", "tagname"
0162             from CMS_COND_31X_GLOBALTAG.TAGINVENTORY_TABLE
0163             where "tagid" in (
0164                 select "tagid"
0165                 from CMS_COND_31X_GLOBALTAG.TAGTREE_TABLE_%s
0166             )
0167         ''' % gt)
0168
0169         for account, tag in cursor:
0170             account_tags.setdefault(account, set([])).add(tag)
0171
0172     for account in sorted(account_tags):
0173         print(account)
0174         for tag in sorted(account_tags[account]):
0175             print('   ', tag)
0176         print()
0177
0178
0179 def check_and_run(args):
0180     if 'SCRAM_ARCH' not in os.environ:
0181         raise Exception('SCRAM_ARCH needs to be set: run cmsenv within a newish release.')
0182
0183     if 'CMSSW_BASE' not in os.environ:
0184         raise Exception('CMSSW_BASE needs to be set: run cmsenv within a newish release.')
0185
0186     if 'jobs' in args and args.jobs <= 0:
0187         raise Exception('If set, --jobs needs to be >= 1.')
0188
0189     aliases = {
0190         'root': 'oracle://cms_orcoff_prep/CMS_R5_CONDITIONS',
0191         'boost': 'oracle://cms_orcoff_prep/CMS_CONDITIONS',
0192     }
0193
0194     if args.db in aliases:
0195         args.db = aliases[args.db]
0196
0197     # Check that the release and database match to prevent mistakes...
0198     if args.db == 'oracle://cms_orcoff_prep/CMS_CONDITIONS' and \
0199         not 'BOOST' in os.environ['CMSSW_VERSION']:
0200         raise Exception('Boost database without a Boost release -- mistake?')
0201
0202     if args.db == 'oracle://cms_orcoff_prep/CMS_R5_CONDITIONS' and \
0203         'BOOST' in os.environ['CMSSW_VERSION']:
0204         raise Exception('ROOT database with a Boost release -- mistake?')
0205
0206     # Create output log folder
0207     os.makedirs(args.output)
0208
0209     args.func(args)
0210
0211
0212 def main():
0213     '''Entry point.
0214     '''
0215
0216     parser = argparse.ArgumentParser(description='conddb_migrate - the CMS Conditions DB migration script')
0217     parser.add_argument('--verbose', '-v', action='count', help='Verbosity level. -v prints debugging information of this tool, like tracebacks in case of errors.')
0218     parser.add_argument('--output', '-o', default=time.strftime('%Y-%m-%d-%H-%M-%S'), help='Output folder. Default: {current_timestamp}, i.e. %(default)s')
0219     parser.add_argument('db', help='Destination database. Aliases: "root" (CMS_CONDITIONS), "boost" (CMS_TEST_CONDITIONS), both in prep. *Make sure the database kind matches the code, i.e. use a BOOST IB when uploading to a Boost database; and a normal release when uploading to the ROOT database -- this script checks the CMSSW_VERSION when using the two official aliases in prep to prevent mistakes, but not for other databases.*')
0220     parser_subparsers = parser.add_subparsers(title='Available subcommands')
0221
0222     parser_account = parser_subparsers.add_parser('account', description='Migrates all (non-migrated) tags, IOVs and payloads, from an account.')
0223     parser_account.add_argument('account', help='The account to migrate.')
0224     parser_account.set_defaults(func=migrate_account)
0225
0226     parser_accounts = parser_subparsers.add_parser('accounts', description='Migrates all accounts (see "account" command).')
0227     parser_accounts.add_argument('--jobs', '-j', type=int, default=4, help='Number of jobs.')
0228     parser_accounts.set_defaults(func=migrate_accounts)
0229
0230     parser_gt = parser_subparsers.add_parser('gt', description='Migrates a single global tag.')
0231     parser_gt.add_argument('gt', help='The global tag to migrate.')
0232     parser_gt.set_defaults(func=migrate_gt)
0233
0234     parser_gts = parser_subparsers.add_parser('gts', description='Migrates all global tags (see "gt" command).')
0235     parser_gts.add_argument('authpath', help='Authentication path.')
0236     parser_gts.add_argument('--jobs', '-j', type=int, default=4, help='Number of jobs.')
0237     parser_gts.set_defaults(func=migrate_gts)
0238
0239     parser_tags_in_gts = parser_subparsers.add_parser('tags_in_gts', description='Dumps the set of tags (including account name) which are in each global tag.')
0240     parser_tags_in_gts.add_argument('authpath', help='Authentication path.')
0241     parser_tags_in_gts.set_defaults(func=tags_in_gts)
0242
0243     args = parser.parse_args()
0244
0245     logging.basicConfig(
0246         format = '[%(asctime)s] %(levelname)s: %(message)s',
0247         level = logging.DEBUG if args.verbose >= 1 else logging.INFO,
0248     )
0249
0250     if args.verbose >= 1:
0251         # Include the traceback
0252         check_and_run(args)
0253     else:
0254         # Only one error line
0255         try:
0256             check_and_run(args)
0257         except Exception as e:
0258             logging.error(e)
0259             sys.exit(1)
0260
0261
0262 if __name__ == '__main__':
0263     main()
0264