Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2021-07-20 22:46:15

0001 #!/usr/bin/env python3
0002 """
0003 Primary Author:
0004 Joshua Dawes - CERN, CMS - The University of Manchester
0005 
0006 Debugging, Integration and Maintenance:
0007 Andres Cardenas - CERN, CMS - Universidad San Francisco
0008 
0009 Upload script wrapper - controls the automatic update system.
0010 
0011 Note: the name of the file follows a different convention to the others because it should be the same as the current upload script name.
0012 
0013 Takes user arguments and passes them to the main upload module CondDBFW.uploads, once the correct version exists.
0014 
0015 1. Ask the server corresponding to the database we're uploading to which version of CondDBFW it has (query the /conddbfw_version/ url).
0016 2. Decide which directory that we can write to - either the current local directory, or /tmp/random_string/.
0017 3. Pull the commit returned from the server into the directory from step 2.
0018 4. Invoke the CondDBFW.uploads module with the arguments given to this script.
0019 
0020 """
0021 
0022 __version__ = 1
0023 
0024 try: 
0025     from CondCore.Utilities.CondDBFW.url_query import url_query
0026 except:
0027     print("ERROR: Could not access the url query utiliy. Yoy are probably not in a CMSSW environment.")
0028     exit(-1)
0029 try:
0030     from StringIO import StringIO
0031 except:
0032     pass
0033 import traceback
0034 import sys
0035 import os
0036 import json
0037 import subprocess
0038 import optparse
0039 import netrc
0040 import shutil
0041 import getpass
0042 import errno
0043 import sqlite3
0044 
0045 
0046 horizontal_rule = "="*60
0047 
0048 def run_upload(**parameters):
0049     """
0050     Imports CondDBFW.uploads and runs the upload with the upload metadata obtained.
0051     """
0052     try:
0053         import CondCore.Utilities.CondDBFW.uploads as uploads
0054     except Exception as e:
0055         traceback.print_exc()
0056         exit("CondDBFW or one of its dependencies could not be imported.\n"\
0057             + "If the CondDBFW directory exists, you are likely not in a CMSSW environment.")
0058     # we have CondDBFW, so just call the module with the parameters given in the command line
0059     uploader = uploads.uploader(**parameters)
0060     result = uploader.upload()
0061 
0062 def getInput(default, prompt = ''):
0063     '''Like raw_input() but with a default and automatic strip().
0064     '''
0065 
0066     answer = raw_input(prompt)
0067     if answer:
0068         return answer.strip()
0069 
0070     return default.strip()
0071 
0072 
0073 def getInputWorkflow(prompt = ''):
0074     '''Like getInput() but tailored to get target workflows (synchronization options).
0075     '''
0076 
0077     while True:
0078         workflow = getInput(defaultWorkflow, prompt)
0079 
0080         if workflow in frozenset(['offline', 'hlt', 'express', 'prompt', 'pcl']):
0081             return workflow
0082 
0083         print('Please specify one of the allowed workflows. See above for the explanation on each of them.')
0084 
0085 
0086 def getInputChoose(optionsList, default, prompt = ''):
0087     '''Makes the user choose from a list of options.
0088     '''
0089 
0090     while True:
0091         index = getInput(default, prompt)
0092 
0093         try:
0094             return optionsList[int(index)]
0095         except ValueError:
0096             print('Please specify an index of the list (i.e. integer).')
0097         except IndexError:
0098             print('The index you provided is not in the given list.')
0099 
0100 
0101 def getInputRepeat(prompt = ''):
0102     '''Like raw_input() but repeats if nothing is provided and automatic strip().
0103     '''
0104 
0105     while True:
0106         answer = raw_input(prompt)
0107         if answer:
0108             return answer.strip()
0109 
0110         print('You need to provide a value.')
0111 
0112 def runWizard(basename, dataFilename, metadataFilename):
0113     while True:
0114         print('''\nWizard for metadata for %s
0115 
0116 I will ask you some questions to fill the metadata file. For some of the questions there are defaults between square brackets (i.e. []), leave empty (i.e. hit Enter) to use them.''' % basename)
0117 
0118         # Try to get the available inputTags
0119         try:
0120             dataConnection = sqlite3.connect(dataFilename)
0121             dataCursor = dataConnection.cursor()
0122             dataCursor.execute('select name from sqlite_master where type == "table"')
0123             tables = set(zip(*dataCursor.fetchall())[0])
0124 
0125             # only conddb V2 supported...
0126             if 'TAG' in tables:
0127                 dataCursor.execute('select NAME from TAG')
0128             # In any other case, do not try to get the inputTags
0129             else:
0130                 raise Exception()
0131 
0132             inputTags = dataCursor.fetchall()
0133             if len(inputTags) == 0:
0134                 raise Exception()
0135             inputTags = list(zip(*inputTags))[0]
0136 
0137         except Exception:
0138             inputTags = []
0139 
0140         if len(inputTags) == 0:
0141             print('\nI could not find any input tag in your data file, but you can still specify one manually.')
0142 
0143             inputTag = getInputRepeat(
0144                 '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. BeamSpotObject_ByRun\ninputTag: ')
0145 
0146         else:
0147             print('\nI found the following input tags in your SQLite data file:')
0148             for (index, inputTag) in enumerate(inputTags):
0149                 print('   %s) %s' % (index, inputTag))
0150 
0151             inputTag = getInputChoose(inputTags, '0',
0152                                       '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. 0 (you select the first in the list)\ninputTag [0]: ')
0153 
0154         databases = {
0155             'oraprod': 'oracle://cms_orcon_prod/CMS_CONDITIONS',
0156             'prod': 'oracle://cms_orcon_prod/CMS_CONDITIONS',
0157             'oradev': 'oracle://cms_orcoff_prep/CMS_CONDITIONS',
0158             'prep': 'oracle://cms_orcoff_prep/CMS_CONDITIONS',
0159         }
0160 
0161         destinationDatabase = ''
0162         ntry = 0
0163         print('\nWhich is the destination database where the tags should be exported?')
0164         print('\n%s) %s' % ('oraprod', databases['oraprod']))
0165         print('\n%s) %s' % ('oradev', databases['oradev']))
0166             
0167         while ( destinationDatabase not in databases.values() ): 
0168             if ntry==0:
0169                 inputMessage = \
0170                 '\nPossible choices: oraprod or oradev \ndestinationDatabase: '
0171             elif ntry==1:
0172                 inputMessage = \
0173                 '\nPlease choose one of the two valid destinations: oraprod or oradev \ndestinationDatabase: '
0174             else:
0175                 raise Exception('No valid destination chosen. Bailing out...')
0176             
0177             databaseInput = getInputRepeat(inputMessage).lower()
0178             if databaseInput in databases.keys():
0179                 destinationDatabase = databases[databaseInput]
0180             ntry += 1
0181 
0182         while True:
0183             since = getInput('',
0184                              '\nWhich is the given since? (if not specified, the one from the SQLite data file will be taken -- note that even if specified, still this may not be the final since, depending on the synchronization options you select later: if the synchronization target is not offline, and the since you give is smaller than the next possible one (i.e. you give a run number earlier than the one which will be started/processed next in prompt/hlt/express), the DropBox will move the since ahead to go to the first safe run instead of the value you gave)\ne.g. 1234\nsince []: ')
0185             if not since:
0186                 since = None
0187                 break
0188             else:
0189                 try:
0190                     since = int(since)
0191                     break
0192                 except ValueError:
0193                     print('The since value has to be an integer or empty (null).')
0194 
0195         userText = getInput('',
0196                             '\nWrite any comments/text you may want to describe your request\ne.g. Muon alignment scenario for...\nuserText []: ')
0197 
0198         destinationTags = {}
0199         while True:
0200             destinationTag = getInput('',
0201                                       '\nWhich is the next destination tag to be added (leave empty to stop)?\ne.g. BeamSpotObjects_PCL_byRun_v0_offline\ndestinationTag []: ')
0202             if not destinationTag:
0203                 if len(destinationTags) == 0:
0204                     print('There must be at least one destination tag.')
0205                     continue
0206                 break
0207 
0208             if destinationTag in destinationTags:
0209                 print(
0210                     'You already added this destination tag. Overwriting the previous one with this new one.')
0211 
0212             destinationTags[destinationTag] = {
0213             }
0214 
0215         metadata = {
0216             'destinationDatabase': destinationDatabase,
0217             'destinationTags': destinationTags,
0218             'inputTag': inputTag,
0219             'since': since,
0220             'userText': userText,
0221         }
0222 
0223         metadata = json.dumps(metadata, sort_keys=True, indent=4)
0224         print('\nThis is the generated metadata:\n%s' % metadata)
0225 
0226         if getInput('n',
0227                     '\nIs it fine (i.e. save in %s and *upload* the conditions if this is the latest file)?\nAnswer [n]: ' % metadataFilename).lower() == 'y':
0228             break
0229     print('Saving generated metadata in %s...'% metadataFilename)
0230     with open(metadataFilename, 'wb') as metadataFile:
0231         metadataFile.write(metadata)
0232 
0233 def parse_arguments():
0234     # read in command line arguments, and build metadata dictionary from them
0235     parser = optparse.OptionParser(description="CMS Conditions Upload Script in CondDBFW.",
0236         usage = 'Usage: %prog [options] <file>')
0237 
0238     # metadata arguments
0239     parser.add_option("-i", "--inputTag", type=str,\
0240                         help="Tag to take IOVs + Payloads from in --sourceDB.")
0241     parser.add_option("-t", "--destinationTag", type=str,\
0242                         help="Tag to copy IOVs + Payloads to in --destDB.")
0243     parser.add_option("-D", "--destinationDatabase", type=str,\
0244                         help="Database to copy IOVs + Payloads to.")
0245     parser.add_option("-s", "--since", type=int,\
0246                         help="Since to take IOVs from.")
0247     parser.add_option("-u", "--userText", type=str,\
0248                         help="Description of --destTag (can be empty).")
0249 
0250     # non-metadata arguments
0251     parser.add_option("-m", "--metadataFile", type=str, help="Metadata file to take metadata from.")
0252 
0253     parser.add_option("-d", "--debug", action="store_true", default=False)
0254     parser.add_option("-v", "--verbose", action="store_true", default=False)
0255     parser.add_option("-T", "--testing", action="store_true")
0256     parser.add_option("--fcsr-filter", type=str, help="Synchronization to take FCSR from for local filtering of IOVs.")
0257 
0258     parser.add_option("-n", "--netrc", help = 'The netrc host (machine) from where the username and password will be read.')
0259     
0260     parser.add_option("-a", "--authPath", help = 'The path of the .netrc file for the authentication. Default: $HOME')
0261 
0262     parser.add_option("-H", "--hashToUse")
0263 
0264     parser.add_option("-S", "--server")
0265 
0266     parser.add_option("-o", "--review-options", action="store_true")
0267 
0268     parser.add_option("-r", "--replay-file")
0269 
0270     (command_line_data, arguments) = parser.parse_args()
0271 
0272     if len(arguments) < 1:
0273         if command_line_data.hashToUse == None:
0274             parser.print_help()
0275             exit(-2)
0276     
0277     command_line_data.sourceDB = arguments[0]
0278 
0279     if command_line_data.replay_file:
0280         dictionary = json.loads("".join(open(command_line_data.replay_file, "r").readlines()))
0281         command_line_data.tier0_response = dictionary["tier0_response"]
0282 
0283     # default is the production server, which can point to either database anyway
0284     server_alias_to_url = {
0285         "prep" : "https://cms-conddb-dev.cern.ch/cmsDbCondUpload/",
0286         "dev" : "https://cms-conddb-dev.cern.ch/cmsDbCondUpload/",
0287         "prod" : "https://cms-conddb.cern.ch/cmsDbCondUpload/"
0288     }
0289 
0290     # if prep, prod or None were given, convert to URLs in dictionary server_alias_to_url
0291     # if not, assume a URL has been given and use this instead
0292     if command_line_data.server in server_alias_to_url.keys():
0293         command_line_data.server = server_alias_to_url[command_line_data.server]
0294 
0295     # resolve destination databases
0296     database_alias_to_connection = {
0297         "prep": "oracle://cms_orcoff_prep/CMS_CONDITIONS",
0298         "dev": "oracle://cms_orcoff_prep/CMS_CONDITIONS",
0299         "prod": "oracle://cms_orcon_adg/CMS_CONDITIONS"
0300     }
0301     
0302     if command_line_data.destinationDatabase in database_alias_to_connection.keys():
0303         command_line_data.destinationDatabase = database_alias_to_connection[command_line_data.destinationDatabase]
0304 
0305 
0306     # use netrc to get username and password
0307     try:
0308         netrc_file = command_line_data.netrc
0309         auth_path = command_line_data.authPath
0310         if not auth_path is None:
0311             if netrc_file is None:
0312                 netrc_file = os.path.join(auth_path,'.netrc')
0313             else:
0314                 netrc_file = os.path.join(auth_path, netrc_file)
0315 
0316         netrc_authenticators = netrc.netrc(netrc_file).authenticators("ConditionUploader")
0317         if netrc_authenticators == None:
0318             print("Your netrc file must contain the key 'ConditionUploader'.")
0319             manual_input = raw_input("Do you want to try to type your credentials? ")
0320             if manual_input == "y":
0321                 # ask for username and password
0322                 username = raw_input("Username: ")
0323                 password = getpass.getpass("Password: ")
0324             else:
0325                 exit()
0326         else:
0327             username = netrc_authenticators[0]
0328             password = netrc_authenticators[2]
0329     except:
0330         print("Couldn't obtain your credentials (either from netrc or manual input).")
0331         exit()
0332 
0333     command_line_data.username = username
0334     command_line_data.password = password
0335     # this will be used as the final destinationTags value by all input methods
0336     # apart from the metadata file
0337     command_line_data.destinationTags = {command_line_data.destinationTag:{}}
0338 
0339     """
0340     Construct metadata_dictionary:
0341     Currently, this is 3 cases:
0342 
0343     1) An IOV is being appended to an existing Tag with an existing Payload.
0344     In this case, we just take all data from the command line.
0345 
0346     2) No metadata file is given, so we assume that ALL upload metadata is coming from the command line.
0347 
0348     3) A metadata file is given, hence we parse the file, and then iterate through command line arguments
0349     since these override the options set in the metadata file.
0350 
0351     """
0352 
0353     # Hash to use, entirely from command line
0354     if command_line_data.hashToUse != None:
0355         command_line_data.userText = ""
0356         metadata_dictionary = command_line_data.__dict__
0357     elif command_line_data.metadataFile == None:
0358         if command_line_data.sourceDB != None and (command_line_data.inputTag == None or command_line_data.destinationTag == None or command_line_data.destinationDatabase == None):
0359             basepath = command_line_data.sourceDB.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
0360             basename = os.path.basename(basepath)
0361             dataFilename = '%s.db' % basepath
0362             metadataFilename = '%s.txt' % basepath
0363 
0364             # Data file
0365             try:
0366                 with open(dataFilename, 'rb') as dataFile:
0367                     pass
0368             except IOError as e:
0369                 errMsg = 'Impossible to open SQLite data file %s' %dataFilename
0370                 print( errMsg )
0371                 ret['status'] = -3
0372                 ret['error'] = errMsg
0373                 return ret
0374 
0375             # Metadata file
0376 
0377             command_line_data.sourceDB = dataFilename
0378 
0379             try:
0380                 with open(metadataFilename, 'rb') as metadataFile:
0381                     pass
0382             except IOError as e:
0383                 if e.errno != errno.ENOENT:
0384                     errMsg = 'Impossible to open file %s (for other reason than not existing)' %metadataFilename
0385                     ret = {}
0386                     ret['status'] = -4
0387                     ret['error'] = errMsg
0388                     exit (ret)
0389 
0390                 if getInput('y', '\nIt looks like the metadata file %s does not exist and not enough parameters were received in the command line. Do you want me to create it and help you fill it?\nAnswer [y]: ' % metadataFilename).lower() != 'y':
0391                     errMsg = 'Metadata file %s does not exist' %metadataFilename
0392                     ret = {}
0393                     ret['status'] = -5
0394                     ret['error'] = errMsg
0395                     exit(ret)
0396                 # Wizard
0397                 runWizard(basename, dataFilename, metadataFilename)
0398             command_line_data.metadataFile = metadataFilename
0399         else:
0400             command_line_data.userText = command_line_data.userText\
0401                                         if command_line_data.userText != None\
0402                                         else str(raw_input("Tag's description [can be empty]:"))
0403             metadata_dictionary = command_line_data.__dict__
0404 
0405     if command_line_data.metadataFile != None:
0406         metadata_dictionary = json.loads("".join(open(os.path.abspath(command_line_data.metadataFile), "r").readlines()))
0407         metadata_dictionary["username"] = username
0408         metadata_dictionary["password"] = password
0409         metadata_dictionary["userText"] = metadata_dictionary.get("userText")\
0410                                             if metadata_dictionary.get("userText") != None\
0411                                             else str(raw_input("Tag's description [can be empty]:"))
0412 
0413         # go through command line options and, if they are set, overwrite entries
0414         for (option_name, option_value) in command_line_data.__dict__.items():
0415             # if the metadata_dictionary sets this, overwrite it
0416             if option_name != "destinationTags":
0417                 if option_value != None or (option_value == None and not(option_name in metadata_dictionary.keys())):
0418                     # if option_value has a value, override the metadata file entry
0419                     # or if option_value is None but the metadata file doesn't give a value,
0420                     # set the entry to None as well
0421                     metadata_dictionary[option_name] = option_value
0422             else:
0423                 if option_value != {None:{}}:
0424                     metadata_dictionary["destinationTags"] = {option_value:{}}
0425                 elif option_value == {None:{}} and not("destinationTags" in metadata_dictionary.keys()):
0426                     metadata_dictionary["destinationTags"] = {None:{}}
0427 
0428     if command_line_data.review_options:
0429         defaults = {
0430             "since" : "Since of first IOV",
0431             "userText" : "Populated by upload process",
0432             "netrc" : "None given",
0433             "fcsr_filter" : "Don't apply",
0434             "hashToUse" : "Using local SQLite file instead"
0435         }
0436         print("Configuration to use for the upload:")
0437         for key in metadata_dictionary:
0438             if not(key) in ["username", "password", "destinationTag"]:
0439                 value_to_print = metadata_dictionary[key] if metadata_dictionary[key] != None else defaults[key]
0440                 print("\t%s : %s" % (key, value_to_print))
0441 
0442         if raw_input("\nDo you want to continue? [y/n] ") != "y":
0443             exit()
0444 
0445     if metadata_dictionary["server"] == None:
0446         if metadata_dictionary["destinationDatabase"] == "oracle://cms_orcoff_prep/CMS_CONDITIONS":
0447             metadata_dictionary["server"] = server_alias_to_url["prep"]
0448         else:
0449             metadata_dictionary["server"] = server_alias_to_url["prod"]
0450 
0451     return metadata_dictionary
0452 
0453 def get_version(url):
0454     query = url_query(url=url + "script_version/")
0455     response = query.send()
0456     return response
0457 
0458 
0459 if __name__ == "__main__":
0460 
0461     upload_metadata = parse_arguments()
0462 
0463     # upload_metadata should be used to decide the service url
0464     final_service_url = upload_metadata["server"]
0465     try:
0466         response = get_version(final_service_url)
0467         server_version = json.loads(response)
0468     except Exception as e:
0469         print(horizontal_rule)
0470         print(e)
0471         print("Could not connect to server at %s"%final_service_url)
0472         print("If you specified a server please check it is correct. If that is not the issue please contact the AlcaDB team.")
0473         print(horizontal_rule)
0474         exit(1)
0475 
0476     if server_version["version"] != __version__:
0477         print(horizontal_rule)
0478         print("Local upload script is different than server version. Please run the following command to get the latest script.")
0479         print("curl --insecure -o uploadConditions.py %sget_upload_script/ && chmod +x uploadConditions.py;"%final_service_url)
0480         print(horizontal_rule)
0481         exit(1)
0482 
0483     import CondCore.Utilities.CondDBFW.data_sources as data_sources
0484 
0485     upload_metadata["sqlite_file"] = upload_metadata.get("sourceDB")
0486 
0487     try:
0488         os.mkdir('upload_logs')
0489     except OSError as e:
0490         pass
0491 
0492     # make new dictionary, and copy over everything except "metadata_source"
0493     upload_metadata_argument = {}
0494     for (key, value) in upload_metadata.items():
0495         if key != "metadata_source":
0496             upload_metadata_argument[key] = value
0497 
0498     upload_metadata["metadata_source"] = data_sources.json_data_node.make(upload_metadata_argument)
0499     try:
0500         # pass dictionary as arguments to match keywords - the constructor has a **kwargs parameter to deal with stray arguments
0501         run_upload(**upload_metadata)
0502         print(horizontal_rule)
0503         print("Process completed without issues. Please check logs for further details.")
0504         print(horizontal_rule)
0505     except SystemExit as e:
0506         print(horizontal_rule)
0507         print("Process exited abnormally. Please check logs for details.")
0508         print(horizontal_rule)
0509         exit(1)
0510     exit(0)