Utilities/python/conddb_migrate.py

0001 #!/usr/bin/env python3
0002 '''CMS Conditions DB migration script.
0003 '''
0004 from __future__ import print_function
0005
0006
0007 import os
0008 import sys
0009 import logging
0010 import argparse
0011 import subprocess
0012 import time
0013 import multiprocessing
0014
0015 import cx_Oracle
0016
0017
0018 accounts = [
0019     #'CMS_COND_TEMP', # not in payloadInspector
0020     'CMS_COND_31X_ALIGNMENT',
0021     'CMS_COND_31X_BEAMSPOT',
0022     'CMS_COND_31X_BTAU',
0023     'CMS_COND_31X_CSC',
0024     'CMS_COND_31X_DQM_SUMMARY',
0025     'CMS_COND_31X_DT',
0026     'CMS_COND_31X_ECAL',
0027     'CMS_COND_31X_FROM21X',
0028     'CMS_COND_31X_GEOMETRY',
0029     'CMS_COND_31X_HCAL',
0030     'CMS_COND_31X_HLT',
0031     'CMS_COND_31X_L1T',
0032     'CMS_COND_31X_PHYSICSTOOLS',
0033     'CMS_COND_31X_PIXEL',
0034     'CMS_COND_31X_PRESHOWER',
0035     'CMS_COND_31X_RPC',
0036     'CMS_COND_31X_RUN_INFO',
0037     'CMS_COND_31X_STRIP',
0038     'CMS_COND_34X_DQM',
0039     'CMS_COND_34X_ECAL',
0040     'CMS_COND_34X_ECAL_PED',
0041     'CMS_COND_34X_GEOMETRY',
0042     'CMS_COND_36X_RPC',
0043     'CMS_COND_38X_HCAL',
0044     #'CMS_COND_38X_PIXEL', # not in payloadInspector
0045     'CMS_COND_39X_PRESHOWER',
0046     #'CMS_COND_310X_ALIGN', # FIXME: Segmentation fault
0047     'CMS_COND_310X_CSC',
0048     'CMS_COND_310X_ECAL_PED',
0049     'CMS_COND_311X_ECAL_LAS',
0050     'CMS_COND_311X_PRESH',
0051     'CMS_COND_42X_DQM',
0052     'CMS_COND_42X_ECAL_LAS',
0053     'CMS_COND_42X_ECAL_LASP',
0054     'CMS_COND_42X_GEOMETRY',
0055     'CMS_COND_42X_HCAL',
0056     'CMS_COND_42X_RUN_INFO',
0057     'CMS_COND_43X_ECAL',
0058     #'CMS_COND_43X_RPC_NOISE', # not in payloadInspector
0059     'CMS_COND_44X_ECAL',
0060     'CMS_COND_44X_GEOMETRY',
0061     'CMS_COND_44X_HCAL',
0062     'CMS_COND_44X_PHYSICSTOOLS',
0063     #'CMS_COND_44X_RPC', # not in payloadInspector
0064     'CMS_COND_ALIGN_000',
0065     'CMS_COND_BEAMSPOT_000',
0066     'CMS_COND_BTAU_000',
0067     'CMS_COND_CSC_000',
0068     'CMS_COND_DQM_000',
0069     'CMS_COND_DT_000',
0070     'CMS_COND_ECAL_000',
0071     'CMS_COND_ECAL_LAS_000',
0072     'CMS_COND_ECAL_PED_000',
0073     'CMS_COND_GEOMETRY_000',
0074     'CMS_COND_HCAL_000',
0075     'CMS_COND_HLT_000',
0076     'CMS_COND_L1T_000',
0077     'CMS_COND_MC_000',
0078     'CMS_COND_PAT_000',
0079     'CMS_COND_PIXEL_000',
0080     'CMS_COND_PRESH_000',
0081     'CMS_COND_RPC_000',
0082     'CMS_COND_RUNINFO_000',
0083     'CMS_COND_STRIP_000',
0084 ]
0085
0086
0087 def run_command(command, output_file):
0088     command = '%s > %s 2>&1' % (command, output_file)
0089     logging.info('Running %s', command)
0090     try:
0091         subprocess.check_call(command, shell=True)
0092     except subprocess.CalledProcessError as e:
0093         logging.error('Error while running %s: return code %s', command, e.returncode)
0094
0095
0096 def migrate_account(args):
0097     command_template = '$CMSSW_BASE/bin/$SCRAM_ARCH/conddb_migrate -s oracle://cms_orcon_adg/%s -d %s'
0098     command = command_template % (args.account, args.db)
0099     run_command(command, os.path.join(args.output, args.account))
0100
0101
0102 def migrate_accounts(args):
0103     def _make_args(args, account):
0104         newargs = argparse.Namespace(**vars(args))
0105         newargs.account = account
0106         return newargs
0107
0108     multiprocessing.Pool(args.jobs).map(migrate_account, [_make_args(args, account) for account in accounts])
0109
0110
0111 def migrate_gt(args):
0112     command_template = '$CMSSW_BASE/bin/$SCRAM_ARCH/conddb_migrate_gt -s oracle://cms_orcon_adg/CMS_COND_31X_GLOBALTAG -d %s -g %s'
0113     command = command_template % (args.db, args.gt)
0114     run_command(command, os.path.join(args.output, args.gt))
0115
0116
0117 def make_gt_connection(args):
0118     logging.info('Fetching global tag list...')
0119     password = subprocess.check_output('''cat %s | grep -F 'CMS_COND_31X_GLOBALTAG' -2 | tail -1 | cut -d'"' -f4''' % os.path.join(args.authpath, 'readOnlyProd.xml'), shell=True).strip()
0120     return cx_Oracle.connect('CMS_COND_GENERAL_R', password, 'cms_orcon_adg')
0121
0122
0123 def fetch_gts(connection):
0124     logging.info('Fetching global tag list...')
0125     cursor = connection.cursor()
0126     cursor.execute('''
0127         select substr(table_name, length('tagtree_table_') + 1) gt
0128         from all_tables
0129         where owner = 'CMS_COND_31X_GLOBALTAG'
0130             and table_name like 'TAGTREE_TABLE_%'
0131         order by gt
0132     ''')
0133     gts = zip(*cursor.fetchall())[0]
0134     logging.info('Fetching global tag list... Done: %s global tags found.', len(gts))
0135     return gts
0136
0137
0138 def migrate_gts(args):
0139     gts = fetch_gts(make_gt_connection(args))
0140
0141     def _make_args(args, gt):
0142         newargs = argparse.Namespace(**vars(args))
0143         newargs.gt = gt
0144         return newargs
0145
0146     multiprocessing.Pool(args.jobs).map(migrate_gt, [_make_args(args, gt) for gt in gts])
0147
0148
0149 def tags_in_gts(args):
0150     # Dynamic SQL is used due to the schema
0151     # This is OK since we trust the input,
0152     # which is the GT database's tables.
0153
0154     connection = make_gt_connection(args)
0155     gts = fetch_gts(connection)
0156     account_tags = {}
0157
0158     for i, gt in enumerate(gts):
0159         logging.info('[%s/%s] Reading %s ...', i+1, len(gts), gt)
0160         cursor = connection.cursor()
0161         cursor.execute('''
0162             select "pfn", "tagname"
0163             from CMS_COND_31X_GLOBALTAG.TAGINVENTORY_TABLE
0164             where "tagid" in (
0165                 select "tagid"
0166                 from CMS_COND_31X_GLOBALTAG.TAGTREE_TABLE_%s
0167             )
0168         ''' % gt)
0169
0170         for account, tag in cursor:
0171             account_tags.setdefault(account, set([])).add(tag)
0172
0173     for account in sorted(account_tags):
0174         print(account)
0175         for tag in sorted(account_tags[account]):
0176             print('   ', tag)
0177         print()
0178
0179
0180 def check_and_run(args):
0181     if 'SCRAM_ARCH' not in os.environ:
0182         raise Exception('SCRAM_ARCH needs to be set: run cmsenv within a newish release.')
0183
0184     if 'CMSSW_BASE' not in os.environ:
0185         raise Exception('CMSSW_BASE needs to be set: run cmsenv within a newish release.')
0186
0187     if 'jobs' in args and args.jobs <= 0:
0188         raise Exception('If set, --jobs needs to be >= 1.')
0189
0190     aliases = {
0191         'root': 'oracle://cms_orcoff_prep/CMS_R5_CONDITIONS',
0192         'boost': 'oracle://cms_orcoff_prep/CMS_CONDITIONS',
0193     }
0194
0195     if args.db in aliases:
0196         args.db = aliases[args.db]
0197
0198     # Check that the release and database match to prevent mistakes...
0199     if args.db == 'oracle://cms_orcoff_prep/CMS_CONDITIONS' and \
0200         not 'BOOST' in os.environ['CMSSW_VERSION']:
0201         raise Exception('Boost database without a Boost release -- mistake?')
0202
0203     if args.db == 'oracle://cms_orcoff_prep/CMS_R5_CONDITIONS' and \
0204         'BOOST' in os.environ['CMSSW_VERSION']:
0205         raise Exception('ROOT database with a Boost release -- mistake?')
0206
0207     # Create output log folder
0208     os.makedirs(args.output)
0209
0210     args.func(args)
0211
0212
0213 def main():
0214     '''Entry point.
0215     '''
0216
0217     parser = argparse.ArgumentParser(description='conddb_migrate - the CMS Conditions DB migration script')
0218     parser.add_argument('--verbose', '-v', action='count', help='Verbosity level. -v prints debugging information of this tool, like tracebacks in case of errors.')
0219     parser.add_argument('--output', '-o', default=time.strftime('%Y-%m-%d-%H-%M-%S'), help='Output folder. Default: {current_timestamp}, i.e. %(default)s')
0220     parser.add_argument('db', help='Destination database. Aliases: "root" (CMS_CONDITIONS), "boost" (CMS_TEST_CONDITIONS), both in prep. *Make sure the database kind matches the code, i.e. use a BOOST IB when uploading to a Boost database; and a normal release when uploading to the ROOT database -- this script checks the CMSSW_VERSION when using the two official aliases in prep to prevent mistakes, but not for other databases.*')
0221     parser_subparsers = parser.add_subparsers(title='Available subcommands')
0222
0223     parser_account = parser_subparsers.add_parser('account', description='Migrates all (non-migrated) tags, IOVs and payloads, from an account.')
0224     parser_account.add_argument('account', help='The account to migrate.')
0225     parser_account.set_defaults(func=migrate_account)
0226
0227     parser_accounts = parser_subparsers.add_parser('accounts', description='Migrates all accounts (see "account" command).')
0228     parser_accounts.add_argument('--jobs', '-j', type=int, default=4, help='Number of jobs.')
0229     parser_accounts.set_defaults(func=migrate_accounts)
0230
0231     parser_gt = parser_subparsers.add_parser('gt', description='Migrates a single global tag.')
0232     parser_gt.add_argument('gt', help='The global tag to migrate.')
0233     parser_gt.set_defaults(func=migrate_gt)
0234
0235     parser_gts = parser_subparsers.add_parser('gts', description='Migrates all global tags (see "gt" command).')
0236     parser_gts.add_argument('authpath', help='Authentication path.')
0237     parser_gts.add_argument('--jobs', '-j', type=int, default=4, help='Number of jobs.')
0238     parser_gts.set_defaults(func=migrate_gts)
0239
0240     parser_tags_in_gts = parser_subparsers.add_parser('tags_in_gts', description='Dumps the set of tags (including account name) which are in each global tag.')
0241     parser_tags_in_gts.add_argument('authpath', help='Authentication path.')
0242     parser_tags_in_gts.set_defaults(func=tags_in_gts)
0243
0244     args = parser.parse_args()
0245
0246     logging.basicConfig(
0247         format = '[%(asctime)s] %(levelname)s: %(message)s',
0248         level = logging.DEBUG if args.verbose >= 1 else logging.INFO,
0249     )
0250
0251     if args.verbose >= 1:
0252         # Include the traceback
0253         check_and_run(args)
0254     else:
0255         # Only one error line
0256         try:
0257             check_and_run(args)
0258         except Exception as e:
0259             logging.error(e)
0260             sys.exit(1)
0261
0262
0263 if __name__ == '__main__':
0264     main()
0265