python/CondDBFW/command_line.py

0001 #!/usr/bin/env python3
0002 """
0003 Command line module that the "command line" script.
0004
0005 Works by taking the main keyword (first command given to the script),
0006 passing that to the function that will deal with that action, along with the following arguments as parameters for that function.
0007 """
0008
0009 from . import querying
0010 import argparse
0011 import datetime
0012
0013 def list_object(arguments):
0014
0015     # set up connection
0016     connection = querying.connect(arguments.db, secrets=arguments.secrets, mode=arguments.mode)
0017
0018     options = ["tag", "gt", "gts_for_tag"]
0019     number_of_options_given = 0
0020     for option in options:
0021         if getattr(arguments, option):
0022             number_of_options_given += 1
0023     if number_of_options_given != 1:
0024         print("You must specify a single object to list.")
0025         exit()
0026
0027     if arguments.tag:
0028         tag_name = arguments.tag
0029         tag = connection.tag(name=tag_name)
0030         if tag:
0031             iovs = tag.iovs(amount=arguments.limit)
0032             iovs.as_table()
0033         else:
0034             print("The Tag '%s' was not found in the database '%s'." % (tag_name, arguments.db))
0035             exit()
0036
0037     elif arguments.gt:
0038         gt_name = arguments.gt
0039         gt = connection.global_tag(name=gt_name)
0040         if gt:
0041             gt_maps = gt.tags(amount=arguments.limit)
0042             gt_maps.as_table(hide=["global_tag_name"])
0043         else:
0044             print("The Global Tag '%s' was not found in the database '%s'." % (gt_name, arguments.db))
0045             exit()
0046
0047     elif arguments.gts_for_tag:
0048         tag_name = arguments.gts_for_tag
0049         tag = connection.tag(name=tag_name)
0050         gts = tag.parent_global_tags(amount=arguments.limit)
0051         gts.as_table(columns=["name", "insertion_time", "snapshot_time"])
0052
0053 def diff_of_tags(arguments):
0054     # get a CondDBFW Tag object for the first tag
0055     # then use the diff() method to draw the table of differences
0056
0057     # set up connection
0058     connection = querying.connect(arguments.db, secrets=arguments.secrets, mode=arguments.mode)
0059
0060     tag1 = connection.tag(name=arguments.tag1)
0061     tag2 = connection.tag(name=arguments.tag2)
0062
0063     tag1.diff(tag2).as_table(columns=["since", arguments.tag1, arguments.tag2])
0064
0065 def diff_of_gts(arguments):
0066     # get a CondDBFW Global Tag object for the first GT
0067     # then use the diff() method to draw the table of differences
0068
0069     # set up connection
0070     connection = querying.connect(arguments.db, secrets=arguments.secrets, mode=arguments.mode)
0071
0072     gt1 = connection.global_tag(name=arguments.gt1)
0073     gt2 = connection.global_tag(name=arguments.gt2)
0074
0075     gt1.diff(gt2).as_table(columns=["Record", "Label", "%s Tag" % arguments.gt1, "%s Tag" % arguments.gt2])
0076
0077 def search(arguments):
0078
0079     raise NotImplementedError("Todo")
0080
0081     connection = querying.connect(arguments.db, secrets=arguments.secrets, mode=arguments.mode)
0082
0083     search_string = connection.regexp(".*%s.*" % arguments.string)
0084
0085 def copy_tag(arguments):
0086
0087     # set up connection
0088     source_connection = querying.connect(arguments.db, secrets=arguments.secrets, mode=arguments.mode, map_blobs=True)
0089     dest_connection = querying.connect(arguments.dest_db, secrets=arguments.secrets, mode=arguments.mode, map_blobs=True)
0090
0091     # get tag from the source database, adjust it, and copy it (with the defined IOV range) to the destination database
0092
0093     print("Reading source Tag.")
0094     source_tag = source_connection.tag(name=arguments.input_tag)
0095     if source_tag == None:
0096         raise Exception("Source Tag doesn't exist.")
0097
0098     # get all IOVs within the range [start, end]
0099     print("Reading source IOVs.")
0100     since_range = source_connection.range(arguments.start, arguments.end)
0101     source_iovs = source_tag.iovs(since=since_range).data()
0102
0103     # get hashes of all IOVs contained in the Tag in the source database
0104     print("Reading source Payloads.")
0105     hashes = source_tag.iovs().get_members("payload_hash").data()
0106     payloads = source_connection.payload(hash=hashes)
0107
0108     print("Writing to destination database...")
0109
0110     # set end_of_validity to -1 because sqlite doesn't support long ints
0111     source_tag.end_of_validity = -1
0112     source_tag.name = arguments.dest_tag
0113     source_tag.modification_time = datetime.datetime.utcnow()
0114
0115     # create new iovs
0116     new_iovs = []
0117     for iov in source_iovs:
0118         new_iovs.append(dest_connection.models["iov"](iov.as_dicts(convert_timestamps=False), convert_timestamps=False))
0119
0120     # write new tag to destination database
0121     print("Writing destination Tag.")
0122     if dest_connection.tag(name=arguments.dest_tag) != None:
0123         dest_connection.write_and_commit(source_tag)
0124
0125     # write new iovs
0126     print("Writing IOVs to destination Tag.")
0127     for iov in new_iovs:
0128         if dest_connection.iov(tag_name=iov.tag_name, since=iov.since, insertion_time=iov.insertion_time) == None:
0129             dest_connection.write_and_commit(iov)
0130
0131     # get payloads used by IOVs and copy those over
0132     print("Copying Payloads over.")
0133     for payload in payloads:
0134         if dest_connection.payload(hash=payload.hash) == None:
0135             dest_connection.write_and_commit(payload)
0136
0137     print("Copy complete.")
0138
0139 def copy_global_tag(arguments):
0140     raise NotImplementedError("Copying Global Tags is currently not supported for this transition command-line interface for CondDBFW.")
0141
0142     # set up connection
0143     source_connection = querying.connect(arguments.db, secrets=arguments.secrets, mode=arguments.mode, map_blobs=True)
0144     dest_connection = querying.connect(arguments.dest_db, secrets=arguments.secrets, mode=arguments.mode, map_blobs=True)
0145
0146     # get CondDBFW Global Tag object
0147     global_tag = source_connection.global_tag(name=arguments.input_gt)
0148     if global_tag == None:
0149         raise Exception("Source Global Tag doesn't exist.")
0150
0151     tag_names = global_tag.tags().get_members("tag_name").data()
0152     tags = source_connection.tag(name=tags)
0153
0154     # copy global tag first
0155     global_tag.insertion_time = datetime.datetime.utcnow()
0156     global_tag.validity = -1
0157     dest_connection.write_and_commit(global_tag)
0158
0159     for tag in tags:
0160         # create temporary argument class
0161         class args(object):
0162             def __init__(self):
0163                 self.input_tag = tag.name
0164                 self.dest_tag = tag.name
0165                 self.start = 1
0166                 self.end = tag.latest_iov()+1
0167                 for attribute in dir(arguments):
0168                     self.__dict__[attribute] = getattr(arguments, attribute)
0169
0170         copy_tag(args())
0171
0172 def parse_command_line(arguments):
0173     """
0174     Assumes script name has been removed from the list of arguments.
0175     Hence, arguments[0] is the subcommand.
0176     """
0177     top_level_parser = argparse.ArgumentParser(description="CondDBFW Command line tool")
0178     top_level_parser.add_argument("--db", type=str, required=False, default="frontier://FrontierProd/CMS_CONDITIONS")
0179     top_level_parser.add_argument("--mode", type=str, required=False, default="w")
0180     top_level_parser.add_argument("--secrets", type=str, required=False)
0181     top_level_parser.add_argument("--limit", type=int, required=False, default=10)
0182
0183     subparser = top_level_parser.add_subparsers(title="Subcommands")
0184
0185     list_parser = subparser.add_parser("list", description="Lists the Metadata objects contained within the given object.")
0186     list_parser.add_argument("--tag", required=False, help="List all IOVs in a Tag.")
0187     list_parser.add_argument("--gt", required=False, help="List all Global Tag Maps linked to a Global Tag.")
0188     list_parser.add_argument("--gts-for-tag", required=False, help="List all Global Tags that contain a Tag.")
0189
0190     list_parser.set_defaults(func=list_object)
0191
0192     diff_parser = subparser.add_parser("diff-tags", description="Gives the differences in payload hashes used by IOVs between Tags.")
0193     diff_parser.add_argument("--tag1", required=True, help="First Tag to use in the comparison.")
0194     diff_parser.add_argument("--tag2", required=True, help="Second Tag to use in the comparison.")
0195
0196     diff_parser.set_defaults(func=diff_of_tags)
0197
0198     gt_diff_parser = subparser.add_parser("diff-gts", description="Gives the differences in Global Tag Maps contained within Global Tag.")
0199     gt_diff_parser.add_argument("--gt1", required=True, help="First Global Tag to use in the comparison.")
0200     gt_diff_parser.add_argument("--gt2", required=True, help="Second Global Tag to use in the comparison.")
0201
0202     gt_diff_parser.set_defaults(func=diff_of_gts)
0203
0204     copy_tag_parser = subparser.add_parser("copy-tag", description="Copies a Tag with its IOVs and Payloads to a destination database."
0205                                                         + "\nFor copying to official databases, use cmsDbCondUpload (https://cms-conddb-dev.cern.ch/cmsDbCondUpload).")
0206     copy_tag_parser.add_argument("--dest-db", required=True, help="Database to copy the Tag and its IOVs to.")
0207     copy_tag_parser.add_argument("--input-tag", required=True, help="Tag to take data from in source database.")
0208     copy_tag_parser.add_argument("--dest-tag", required=True, help="Tag to copy input Tag to in the destination database.")
0209     copy_tag_parser.add_argument("--start", required=True, help="Since to start from.  If this is between two, the highest one is taken (no adjustments are made).")
0210     copy_tag_parser.add_argument("--end", required=True, help="Since to finidh at.  If this is between two, the lowest one is taken (no adjustments are made).")
0211
0212     copy_tag_parser.set_defaults(func=copy_tag)
0213
0214     parsed_arguments = top_level_parser.parse_args()
0215
0216     print("Using database '%s'." % parsed_arguments.db)
0217
0218     parsed_arguments.func(parsed_arguments)
0219
0220 if __name__ == "__main__":
0221     import sys
0222     parse_command_line(sys.argv[1:])