Project CMSSW displayed by LXR CMSSW_13_2_X_2023-05-26-2300/​CondCore/​Utilities/​python/​CondDBFW/​uploads.py	Source navigation Diff markup Identifier search General search
	Version: CMSSW_13_2_X_2023-05-26-2300 CMSSW_13_2_X_2023-05-25-2300 CMSSW_13_2_X_2023-05-24-2300 CMSSW_13_2_X_2023-05-23-2300 CMSSW_13_2_X_2023-05-22-2300 CMSSW_13_2_X_2023-05-21-2300 Revert   Change

File indexing completed on 2023-03-17 10:46:31

 #!/usr/bin/env python3
 """
 
 Joshua Dawes - CERN, CMS - The University of Manchester
 
 This module holds classes to help with uploading conditions to the drop box web service, which also uses CondDBFW to read and write data.
 
 """
 
 import os
 import json
 import base64
 from datetime import datetime
 from urllib.parse import urlencode
 import math
 import sys
 import traceback
 import netrc
 
 from .url_query import url_query
 from . import models
 from . import errors
 from . import data_sources
 from . import querying
 from .errors import *
 from .utils import to_timestamp, to_datetime, friendly_since
 
 def friendly_since(time_type, since):
     """
     Takes a since and, if it is Run-based expressed as Lumi-based, returns the run number.
     Otherwise, returns the since without transformations.
     """
     if time_type == "Run" and (since & 0xffffff) == 0:
         return since >> 32
     else:
         return since
 
 # this is simple, and works for now - if logging requirements change, I will write a logging class to manage logging
 def log(file_handle, message):
     """
     Very simple logging function, used by output class.
     """
     file_handle.write("[%s] %s\n" % (to_timestamp(datetime.utcnow()), message))
 
 def new_log_file_id():
     """
     Find a new client-side log file name.
 
     Note: This cannot use the upload session token since logs need to be written before this is opened.
     However, this can be changed so that the filename that uses the token is written to once
     it is obtained.
     """
     # new id = number of log files + 1
     # (primitive - matching the hash of the upload session may be a better idea)
     log_files = [file for file in os.listdir(os.path.join(os.getcwd(), "upload_logs")) if "upload_log" in file]
     new_id = len(log_files)+1
     return new_id
 
 class output():
     """
     Used to control output to the console and to the client-side log.
     """
 
     def __init__(self, log_handle=None, verbose=False):
         # first time writing progress bar, don't need to go back along the line
         self.current_output_length = 0
         self._verbose = verbose
         self._log_handle = log_handle
 
     def write(self, message="", ignore_verbose=False):
         """
         Write to the console and to the log file held by self.
         """
         if ignore_verbose:
             print(message)
         elif self._verbose:
             print(message)
         if self._log_handle != None:
             log(self._log_handle, message)
 
 class uploader(object):
     """
     Upload session controller - creates, tracks, and deletes upload sessions on the server.
     """
 
     def __init__(self, metadata_source=None, debug=False, verbose=False, testing=False, server="https://cms-conddb-dev.cern.ch/cmsDbCondUpload/", **kwargs):
         """
         Upload constructor:
         Given an SQLite file and a Metadata sources, reads into a dictionary read for it to be encoded and uploaded.
 
         Note: kwargs is used to capture stray arguments - arguments that do not match keywords will not be used.
 
         Note: default value of service_url should be changed for production.
         """
         # set private variables
         self._debug = debug
         self._verbose = verbose
         self._testing = testing
         # initialise server-side log data as empty string - will be replaced when we get a response back from the server
         self._log_data = ""
         self._SERVICE_URL = server
         self.upload_session_id = None
 
         # set up client-side log file
         self.upload_log_file_name = "upload_logs/upload_log_%d" % new_log_file_id()
         self._handle = open(self.upload_log_file_name, "a")
 
         # set up client-side logging object
         self._outputter = output(verbose=verbose, log_handle=self._handle)
         self._outputter.write("Using server instance at '%s'." % self._SERVICE_URL)
 
         # expect a CondDBFW data_source object for metadata_source
         if metadata_source == None:
             # no upload metadat has been given - we cannot continue with the upload
             self.exit_upload("A source of metadata must be given so CondDBFW knows how to upload conditions.")
         else:
             # set up global metadata source variable
             self.metadata_source = metadata_source.data()
 
         # check for the destination tag
         # this is required whatever type of upload we're performing
         if self.metadata_source.get("destinationTags") == None:
             self.exit_upload("No destination Tag was given.")
         else:
             if type(self.metadata_source.get("destinationTags")) == dict and list(self.metadata_source.get("destinationTags").keys())[0] == None:
                 self.exit_upload("No destination Tag was given.")
 
         # make sure a destination database was given
         if self.metadata_source.get("destinationDatabase") == None:
             self.exit_upload("No destination database was given.")
 
         # get Conditions metadata
         if self.metadata_source.get("sourceDB") == None and self.metadata_source.get("hashToUse") == None:
             """
             If we have neither an sqlite file nor the command line data
             """
             self.exit_upload("You must give either an SQLite database file, or the necessary command line arguments to replace one."\
                             + "\nSee --help for command line argument information.")
         elif self.metadata_source.get("sourceDB") != None:
             """
             We've been given an SQLite file, so try to extract Conditions Metadata based on that and the Upload Metadata in metadata_source
             We now extract the Tag and IOV data from SQLite.  It is added to the dictionary for sending over HTTPs later.
             """
 
             # make sure we have an input tag to look for in the source db
             self.input_tag = metadata_source.data().get("inputTag")
             if self.input_tag == None:
                 self.exit_upload("No input Tag name was given.")
 
             # set empty dictionary to contain Tag and IOV data from SQLite
             result_dictionary = {}
             self.sqlite_file_name = self.metadata_source["sourceDB"]
             if not(os.path.isfile(self.sqlite_file_name)):
                 self.exit_upload("SQLite file '%s' given doesn't exist." % self.sqlite_file_name)
             sqlite_con = querying.connect("sqlite://%s" % os.path.abspath(self.sqlite_file_name))
 
             self._outputter.write("Getting Tag and IOVs from SQLite database.")
 
             # query for Tag, check for existence, then convert to dictionary
             tag = sqlite_con.tag(name=self.input_tag)
             if tag == None:
                 self.exit_upload("The source Tag '%s' you gave was not found in the SQLite file." % self.input_tag)
             tag = tag.as_dicts(convert_timestamps=True)
 
             # query for IOVs, check for existence, then convert to dictionaries
             iovs = sqlite_con.iov(tag_name=self.input_tag)
             if iovs == None:
                 self.exit_upload("No IOVs found in the SQLite file given for Tag '%s'." % self.input_tag)
             iovs = iovs.as_dicts(convert_timestamps=True)
             iovs = [iovs] if type(iovs) != list else iovs
 
             """
             Finally, get the list of all Payload hashes of IOVs,
             then compute the list of hashes for which there is no Payload for
             this is used later to decide if we can continue the upload if the Payload was not found on the server.
             """
             iovs_for_hashes = sqlite_con.iov(tag_name=self.input_tag)
             if iovs_for_hashes.__class__ == data_sources.json_list:
                 hashes_of_iovs = iovs_for_hashes.get_members("payload_hash").data()
             else:
                 hashes_of_iovs = [iovs_for_hashes.payload_hash]
             self.hashes_with_no_local_payload = [payload_hash for payload_hash in hashes_of_iovs if sqlite_con.payload(hash=payload_hash) == None]
 
             # close session open on SQLite database file
             sqlite_con.close_session()
 
         elif metadata_source.data().get("hashToUse") != None:
             """
             Assume we've been given metadata in the command line (since no sqlite file is there, and we have command line arguments).
             We now use Tag and IOV data from command line.  It is added to the dictionary for sending over HTTPs later.
             """
 
             # set empty dictionary to contain Tag and IOV data from command line
             result_dictionary = {}
 
             now = to_timestamp(datetime.utcnow())
             # tag dictionary will be taken from the server
             # this does not require any authentication
             tag = self.get_tag_dictionary()
             self.check_response_for_error_key(tag)
             iovs = [{"tag_name" : self.metadata_source["destinationTag"], "since" : self.metadata_source["since"], "payload_hash" : self.metadata_source["hashToUse"],\
                     "insertion_time" : now}]
 
             # hashToUse cannot be stored locally (no sqlite file is given), so register it as not found
             self.hashes_with_no_local_payload = [self.metadata_source["hashToUse"]]
 
             # Note: normal optimisations will still take place - since the hash checking stage can tell if hashToUse does not exist on the server side
 
         # if the source Tag is run-based, convert sinces to lumi-based sinces with lumi-section = 0
         if tag["time_type"] == "Run":
             for (i, iov) in enumerate(iovs):
                 iovs[i]["since"] = iovs[i]["since"] << 32
 
         result_dictionary = {"inputTagData" : tag, "iovs" : iovs}
 
         # add command line arguments to dictionary
         # remembering that metadata_source is a json_dict object
         result_dictionary.update(metadata_source.data())
 
         # store in instance variable
         self.data_to_send = result_dictionary
 
         # if the since doesn't exist, take the first since from the list of IOVs
         if result_dictionary.get("since") == None:
             result_dictionary["since"] = sorted(iovs, key=lambda iov : iov["since"])[0]["since"]
         elif self.data_to_send["inputTagData"]["time_type"] == "Run":
             # Tag time_type says IOVs use Runs for sinces, so we convert to Lumi-based for uniform processing
             self.data_to_send["since"] = self.data_to_send["since"] << 32
 
         """
         TODO - Settle on a single destination tag format.
         """
         # look for deprecated metadata entries - give warnings
         # Note - we only really support this format
         try:
             if type(result_dictionary["destinationTags"]) == dict:
                 self._outputter.write("WARNING: Multiple destination tags in a single metadata source is deprecated.")
         except Exception as e:
             self._outputter.write("ERROR: %s" % str(e))
 
     @check_response(check="json")
     def get_tag_dictionary(self):
         url_data = {"tag_name" : self.metadata_source["destinationTag"], "database" : self.metadata_source["destinationDatabase"]}
         request = url_query(url=self._SERVICE_URL + "get_tag_dictionary/", url_data=url_data)
         response = request.send()
         return response
 
     def check_response_for_error_key(self, response_dict, exit_if_error=True):
         """
         Checks the decoded response of an HTTP request to the server.
         If it is a dictionary, and one of its keys is "error", the server returned an error
         """
         # if the decoded response data is a dictionary and has an error key in it, we should display an error and its traceback
         if type(response_dict) == dict and "error" in list(response_dict.keys()):
             splitter_string = "\n%s\n" % ("-"*50)
             self._outputter.write("\nERROR: %s" % splitter_string, ignore_verbose=True)
             self._outputter.write(response_dict["error"], ignore_verbose=True)
 
             # if the user has given the --debug flag, show the traceback as well
             if self._debug:
                 # suggest to the user to email this to db upload experts
                 self._outputter.write("\nTRACEBACK (since --debug is set):%s" % splitter_string, ignore_verbose=True)
                 if response_dict.get("traceback") != None:
                     self._outputter.write(response_dict["traceback"], ignore_verbose=True)
                 else:
                     self._outputter.write("No traceback was returned from the server.", ignore_verbose=True)
             else:
                 self._outputter.write("Use the --debug option to show the traceback of this error.", ignore_verbose=True)
 
             # write server side log to client side (if we have an error from creating an upload session, the log is in its initial state (""))
             # if an error has occurred on the server side, a log will have been written
             self.write_server_side_log(response_dict.get("log_data"))
 
             if exit_if_error:
                 if self._testing:
                     return False
                 else:
                     exit()
         elif not("error" in list(response_dict.keys())) and "log_data" in list(response_dict.keys()):
             # store the log data, if it's there, in memory - this is used if a request times out and we don't get any log data back
             self._log_data = response_dict["log_data"]
             return True
 
     def write_server_side_log(self, log_data):
         """
         Given the log data from the server, write it to a client-side log file.
         """
         # if the server_side_log directory doesn't exist, create it
         # without it we can't write the log when we download it from the server
         if not(os.path.exists(os.path.join(os.getcwd(), "server_side_logs/"))):
             os.makedirs("server_side_logs/")
 
         # directory exists now, write to client-side log file
         server_log_file_name = None
         try:
             # if the upload session does not exist yet, don't try to write the log file
             if self.upload_session_id == None:
                 raise Exception("No upload session")
             # create a write handle to the file, decode the log data from base64, write and close
             server_log_file_name = "server_side_logs/upload_log_%s" % str(self.upload_session_id)
             handle = open(server_log_file_name, "w")
             handle.write(base64.b64decode(log_data))
             handle.close()
         except Exception as e:
             # reset log file name to None so we don't try to write it later
             server_log_file_name = None
             #self._outputter.write("Couldn't write the server-side log file.\nThis may be because no upload session could be opened.")
 
         # tell the user where the log files are
         # in the next iteration we may just merge the log files and store one log (how it's done in the plotter module)
         if server_log_file_name != None:
             print("Log file from server written to '%s'." % server_log_file_name)
         else:
             print("No server log file could be written locally.")
 
         print("Log file from CondDBFW written to '%s'." % self.upload_log_file_name)
 
     def exit_upload(self, message=None):
         """
         Used to exit the script - which only happens if an error has occurred.
         If the --testing flag was passed by the user, we should return False for failure, and not exit
         """
         if self.upload_session_id != None:
             # only try to close the upload session if an upload session has been obtained
             response = self.close_upload_session(self.upload_session_id)
             no_error = self.check_response_for_error_key(response)
             # if no error was found in the upload session closure request,
             # we still have to write the server side log
             if no_error:
                 self.write_server_side_log(self._log_data)
         # close client-side log handle
         self._handle.close()
         if message != None:
             print("\n%s\n" % message)
         if self._testing:
             return False
         else:
             exit()
 
     def upload(self):
         """
         Calls methods that send HTTP requests to the upload server.
         """
 
         """
         Open an upload session on the server - this also gives us a tag lock on the tag being uploaded, if it is available.
         """
         try:
 
             # get upload session, check response for error key
             upload_session_data = self.get_upload_session_id()
             no_error = self.check_response_for_error_key(upload_session_data)
 
             # if there was an error and we're testing, return False for the testing module
             if not(no_error) and self._testing:
                 return False
 
             self.upload_session_id = upload_session_data["id"]
             self._outputter.write("Upload session obtained with token '%s'." % self.upload_session_id)
             self.server_side_log_file = upload_session_data["log_file"]
 
         except errors.NoMoreRetriesException as no_more_retries:
             return self.exit_upload("Ran out of retries opening an upload session, where the limit was 3.")
         except Exception as e:
             # something went wrong that we have no specific exception for, so just exit and output the traceback if --debug is set.
             self._outputter.write(traceback.format_exc(), ignore_verbose=True)
 
             if not(self._verbose):
                 self._outputter.write("Something went wrong that isn't handled by code - to get the traceback, run again with --verbose.")
             else:
                 self._outputter.write("Something went wrong that isn't handled by code - the traceback is above.")
 
             return self.exit_upload()
 
         """
         Only if a value is given for --fcsr-filter, run FCSR filtering on the IOVs locally.
         """
         if self.data_to_send["fcsr_filter"] != None:
             """
             FCSR Filtering:
             Filtering the IOVs before we send them by getting the First Conditions Safe Run
             from the server based on the target synchronization type.
             """
             if self.data_to_send["inputTagData"]["time_type"] != "Time":
                 # if we have a time-based tag, we can't do FCSR validation - this is also the case on the server side
                 try:
                     self.filter_iovs_by_fcsr(self.upload_session_id)
                     # this function does not return a value, since it just operates on data - so no point checking for an error key
                     # the error key check is done inside the function on the response from the server
                 except errors.NoMoreRetriesException as no_more_retries:
                     return self.exit_upload("Ran out of retries trying to filter IOVs by FCSR from server, where the limit was 3.")
                 except Exception as e:
                     # something went wrong that we have no specific exception for, so just exit and output the traceback if --debug is set.
                     self._outputter.write(traceback.format_exc(), ignore_verbose=True)
 
                     if not(self._verbose):
                         self._outputter.write("Something went wrong that isn't handled by code - to get the traceback, run again with --verbose.")
                     else:
                         self._outputter.write("Something went wrong that isn't handled by code - the traceback is above.")
 
                     return self.exit_upload()
             else:
                 self._outputter.write("The Tag you're uploading is time-based, so we can't do any FCSR-based validation.  FCSR filtering is being skipped.")
 
         """
         Check for the hashes that the server doesn't have - only send these (but in the next step).
         """
         try:
 
             check_hashes_response = self.get_hashes_to_send(self.upload_session_id)
             # check for an error key in the response
             no_error = self.check_response_for_error_key(check_hashes_response)
 
             # if there was an error and we're testing, return False for the testing module
             if not(no_error) and self._testing:
                 return False
 
             # finally, check hashes_not_found with hashes not found locally - if there is an intersection, we stop the upload
             # because if a hash is not found and is not on the server, there is no data to upload
             all_hashes = [iov["payload_hash"] for iov in self.data_to_send["iovs"]]
             hashes_not_found = check_hashes_response["hashes_not_found"]
             hashes_found = list(set(all_hashes) - set(hashes_not_found))
             self._outputter.write("Checking for IOVs that have no Payload locally or on the server.")
             # check if any hashes not found on the server is used in the local SQLite database
             for hash_not_found in hashes_not_found:
                 if hash_not_found in self.hashes_with_no_local_payload:
                     return self.exit_upload("IOV with hash '%s' does not have a Payload locally or on the server.  Cannot continue." % hash_not_found)
 
             for hash_found in hashes_found:
                 if hash_found in self.hashes_with_no_local_payload:
                     self._outputter.write("Payload with hash %s on server, so can upload IOV." % hash_found)
 
             self._outputter.write("All IOVs either come with Payloads or point to a Payload already on the server.")
 
         except errors.NoMoreRetriesException as no_more_retries:
             # for now, just write the log if we get a NoMoreRetriesException
             return self.exit_upload("Ran out of retries trying to check hashes of payloads to send, where the limit was 3.")
         except Exception as e:
             # something went wrong that we have no specific exception for, so just exit and output the traceback if --debug is set.
             self._outputter.write(traceback.format_exc(), ignore_verbose=True)
 
             if not(self._verbose):
                 self._outputter.write("Something went wrong that isn't handled by code - to get the traceback, run again with --verbose.")
             else:
                 self._outputter.write("Something went wrong that isn't handled by code - the traceback is above.")
 
             return self.exit_upload()
 
         """
         Send the payloads the server told us about in the previous step (returned from get_hashes_to_send)
         exception handling is done inside this method, since it calls a method itself for each payload.
         """
         send_payloads_response = self.send_payloads(check_hashes_response["hashes_not_found"], self.upload_session_id)
         if self._testing and not(send_payloads_response):
             return False
 
         """
         Final stage - send metadata to server (since the payloads are there now)
         if this is successful, once it finished the upload session is closed on the server and the tag lock is released.
         """
         try:
 
             # note that the response (in send_metadata_response) is already decoded from base64 by the response check decorator
             send_metadata_response = self.send_metadata(self.upload_session_id)
             no_error = self.check_response_for_error_key(send_metadata_response)
             if not(no_error) and self._testing:
                 return False
 
             # we have to call this explicitly here since check_response_for_error_key only writes the log file
             # if an error has occurred, whereas it should always be written here
             self.write_server_side_log(self._log_data)
 
         except errors.NoMoreRetriesException as no_more_retries:
             return self.exit_upload("Ran out of retries trying to send metadata, where the limit was 3.")
         except Exception as e:
             # something went wrong that we have no specific exception for, so just exit and output the traceback if --debug is set.
             self._outputter.write(traceback.format_exc(), ignore_verbose=True)
 
             if not(self._verbose):
                 self._outputter.write("Something went wrong that isn't handled by code - to get the traceback, run again with --verbose.")
             else:
                 self._outputter.write("Something went wrong that isn't handled by code - the traceback is above.")
 
             return self.exit_upload()
 
         # close client side log handle
         self._handle.close()
 
         # if we're running the testing script, return True to say the upload has worked
         if self._testing:
             return True
 
     @check_response(check="json")
     def get_upload_session_id(self):
         """
         Open an upload session on the server, and get a unique token back that we can use to authenticate for all future requests,
         as long as the upload session is still open.
         """
         self._outputter.write("Getting upload session.")
 
         # send password in the body so it can be encrypted over https
         # username and password are taken from the netrc file
         # at this point, the value in username_or_token is always a username, since
         # this method's end result is obtaining a token.
         body_data = base64.b64encode(json.dumps(
                 {
                     "destinationTag" : list(self.data_to_send["destinationTags"].keys())[0],
                     "username_or_token" : self.data_to_send["username"],
                     "password" : self.data_to_send["password"]
                 }
             ).encode('UTF-8'))
 
         url_data = {"database" : self.data_to_send["destinationDatabase"]}
 
         query = url_query(url=self._SERVICE_URL + "get_upload_session/", body=body_data, url_data=url_data)
         response = query.send()
         return response
 
     @check_response(check="json")
     def close_upload_session(self, upload_session_id):
         """
         Close an upload session on the server by calling its close_upload_session end-point.
         This is done if there is an error on the client-side.
         """
         self._outputter.write("An error occurred - closing the upload session on the server.")
         url_data = {"database" : self.data_to_send["destinationDatabase"], "upload_session_id" : upload_session_id}
         query = url_query(url=self._SERVICE_URL + "close_upload_session/", url_data=url_data)
         response = query.send()
         return response
 
     @check_response(check="json")
     def get_fcsr_from_server(self, upload_session_id):
         """
         Execute the HTTPs request to ask the server for the FCSR.
 
         Note: we do this in a separate function we so we can do the decoding check for json data with check_response.
         """
         # tiny amount of client-side logic here - all of the work is done on the server
         # tier0_response uses get() so if the key isn't present, we default to None
         # tier0_response is for replaying uploads from the old upload service, with knowledge of the tier0 response
         # when those uploads happened.
         url_data = {
                         "database" : self.data_to_send["destinationDatabase"],
                         "upload_session_id" : upload_session_id,
                         "destinationTag" : list(self.data_to_send["destinationTags"].keys())[0],
                         "sourceTagSync" : self.data_to_send["fcsr_filter"],
                         "tier0_response" : self.data_to_send.get("tier0_response")
                     }
         query = url_query(url=self._SERVICE_URL + "get_fcsr/", url_data=url_data)
         result = query.send()
         return result
 
     def filter_iovs_by_fcsr(self, upload_session_id):
         """
         Ask for the server for the FCSR based on the synchronization type of the source Tag.
         Then, modify the IOVs (possibly remove some) based on the FCSR we received.
         This is useful in the case that most IOVs have different payloads, and our FCSR is close to the end of the range the IOVs cover.
         """
         self._outputter.write("Getting the First Condition Safe Run for the current sync type.")
 
         fcsr_data = self.get_fcsr_from_server(upload_session_id)
         fcsr = fcsr_data["fcsr"]
         fcsr_changed = fcsr_data["fcsr_changed"]
         new_sync = fcsr_data["new_sync"]
 
         if fcsr_changed:
             self._outputter.write("Synchronization '%s' given was changed to '%s' to match destination Tag." % (self.data_to_send["fcsr_filter"], new_sync))
 
         self._outputter.write("Synchronization '%s' gave FCSR %d for FCSR Filtering."\
                             % (self.data_to_send["fcsr_filter"], friendly_since(self.data_to_send["inputTagData"]["time_type"], fcsr)))
 
         """
         There may be cases where this assumption is not correct (that we can reassign since if fcsr > since)
         Only set since to fcsr from server if the fcsr is further along than the user is trying to upload to
         Note: this applies to run, lumi and timestamp run_types.
         """
 
         # if the fcsr is above the since given by the user, we need to set the user since to the fcsr
         if fcsr > self.data_to_send["since"]:
             # check if we're uploading to offline sync - if so, then user since must be >= fcsr, so we should report an error
             if self.data_to_send["fcsr_filter"].lower() == "offline":
                 self._outputter.write("If you're uploading to offline, you can't upload to a since < FCSR.\nNo upload has been processed.")
                 self.exit_upload()
             self.data_to_send["since"] = fcsr
 
         self._outputter.write("Final FCSR after comparison with FCSR received from server is %d."\
                                 % friendly_since(self.data_to_send["inputTagData"]["time_type"], int(self.data_to_send["since"])))
 
         """
         Post validation processing assuming destination since is now valid.
 
         Because we don't have an sqlite database to query (everything's in a dictionary),
         we have to go through the IOVs manually find the greatest since that's less than
         the destination since.
 
         Purpose of this algorithm: move any IOV sinces that we can use up to the fcsr without leaving a hole in the Conditions coverage
         """
         
         max_since_below_dest = self.data_to_send["iovs"][0]["since"]
         for (i, iov) in enumerate(self.data_to_send["iovs"]):
             if self.data_to_send["iovs"][i]["since"] <= self.data_to_send["since"] and self.data_to_send["iovs"][i]["since"] > max_since_below_dest:
                 max_since_below_dest = self.data_to_send["iovs"][i]["since"]
 
         # only select iovs that have sinces >= max_since_below_dest
         # and then shift any IOVs left to the destination since
         self.data_to_send["iovs"] = [iov for iov in self.data_to_send["iovs"] if iov["since"] >= max_since_below_dest]
         for (i, iov) in enumerate(self.data_to_send["iovs"]):
             if self.data_to_send["iovs"][i]["since"] < self.data_to_send["since"]:
                 self.data_to_send["iovs"][i]["since"] = self.data_to_send["since"]
 
         # modify insertion_time of iovs
         new_time = to_timestamp(datetime.utcnow())
         for (i, iov) in enumerate(self.data_to_send["iovs"]):
             self.data_to_send["iovs"][i]["insertion_time"] = new_time
 
     def get_all_hashes(self):
         """
         Get all the hashes from the dictionary of IOVs we have from the SQLite file.
         """
         self._outputter.write("\tGetting list of all hashes found in SQLite database.")
         hashes = [iov["payload_hash"] for iov in self.data_to_send["iovs"]]
         return hashes
 
     @check_response(check="json")
     def get_hashes_to_send(self, upload_session_id):
         """
         Get the hashes of the payloads we want to send that the server doesn't have yet.
         """
         self._outputter.write("Getting list of hashes that the server does not have Payloads for, to send to server.")
         post_data = json.dumps(self.get_all_hashes())
         url_data = {"database" : self.data_to_send["destinationDatabase"], "upload_session_id" : upload_session_id}
         query = url_query(url=self._SERVICE_URL + "check_hashes/", url_data=url_data, body=post_data)
         response = query.send()
         return response
 
     def send_payloads(self, hashes, upload_session_id):
         """
         Send a list of payloads corresponding to hashes we got from the SQLite file and filtered by asking the server.
         """
         # if we have no hashes, we can't send anything
         # but don't exit since it might mean all the Payloads were already on the server
         if len(hashes) == 0:
             self._outputter.write("No hashes to send - moving to metadata upload.")
             return True
         else:
             self._outputter.write("Sending payloads of hashes not found:")
             # construct connection string for local SQLite database file
             database = ("sqlite://%s" % os.path.abspath(self.sqlite_file_name)) if type(self.sqlite_file_name) == str else self.sqlite_file_name
             # create CondDBFW connection that maps blobs - as we need to query for payload BLOBs (disabled by default in CondDBFW)
             self._outputter.write("\tConnecting to input SQLite database.")
             con = querying.connect(database, map_blobs=True)
 
             # query for the Payloads
             self._outputter.write("\tGetting Payloads from SQLite database based on list of hashes.")
             byte_hashes = [bytes(h, 'utf-8') for h in hashes]
             payloads = con.payload(hash=byte_hashes)
             # if we get a single Payload back, put it in a list and turn it into a json_list
             if payloads and payloads.__class__ != data_sources.json_list:
                 payloads = data_sources.json_data_node.make([payloads])
 
             # close the session with the SQLite database file - we won't use it again
             con.close_session()
 
             # if found some Payloads, send them
             if payloads:
                 # Note: there is an edge case in which the SQLite file could have been queried
                 # to delete the Payloads since we queried it for IOV hashes.  This may be handled in the next iteration.
                 # send http post with data blob in body, and everything else as URL parameters
                 # convert Payload to a dictionary - we can put most of this into the URL of the HTTPs request
                 dicts = payloads.as_dicts()
                 self._outputter.write("Uploading Payload BLOBs:")
 
                 # for each payload, send the BLOB to the server
                 for n, payload in enumerate(dicts):
                     self._outputter.write("\t(%d/%d) Sending payload with hash '%s'." % (n+1, len(dicts), payload["hash"]))
                     response = self.send_blob(payload, upload_session_id)
                     # check response for errors
                     no_error = self.check_response_for_error_key(response, exit_if_error=True)
                     if not(no_error):
                         return False
                     self._outputter.write("\tPayload sent - moving to next one.")
                 self._outputter.write("All Payloads uploaded.")
                 return True
             else:
                 return False
 
     @check_response(check="json")
     def send_blob(self, payload, upload_session_id):
         """
         Send the BLOB of a payload over HTTP.
         The BLOB is put in the request body, so no additional processing has to be done on the server side, apart from decoding from base64.
         """
         # encode the BLOB data of the Payload to make sure we don't send a character that will influence the HTTPs request
         blob_data = base64.b64encode(payload["data"])
 
         url_data = {"database" : self.data_to_send["destinationDatabase"], "upload_session_id" : upload_session_id}
 
         # construct the data to send in the body and header of the HTTPs request
         for key in list(payload.keys()):
             # skip blob
             if key != "data":
                 if key == "insertion_time":
                     url_data[key] = to_timestamp(payload[key])
                 else:
                     url_data[key] = payload[key]
 
         request = url_query(url=self._SERVICE_URL + "store_payload/", url_data=url_data, body=blob_data)
 
         # send the request and return the response
         # Note - the url_query module will handle retries, and will throw a NoMoreRetriesException if it runs out
         try:
             request_response = request.send()
             return request_response
         except Exception as e:
             # make sure we don't try again - if a NoMoreRetriesException has been thrown, retries have run out
             if type(e) == errors.NoMoreRetriesException:
                 self._outputter.write("\t\t\tPayload with hash '%s' was not uploaded because the maximum number of retries was exceeded." % payload["hash"])
                 self._outputter.write("Payload with hash '%s' was not uploaded because the maximum number of retries was exceeded." % payload["hash"])
             return json.dumps({"error" : str(e), "traceback" : traceback.format_exc()})
 
     @check_response(check="json")
     def send_metadata(self, upload_session_id):
         """
         Final part of the upload process - send the Conditions metadata (Tag, IOVs - not upload metadata).
         The server closes the session (and releases the tag lock) after processing has been completed.
         """
 
         # set user text if it's empty
         if self.data_to_send["userText"] in ["", None]:
             self.data_to_send["userText"] = "Tag '%s' uploaded from CondDBFW client." % list(self.data_to_send["destinationTags"].keys())[0]
 
         self._outputter.write("Sending metadata to server - see server_side_log at server_side_logs/upload_log_%s for details on metadata processing on server side."\
                             % self.upload_session_id)
 
         # sent the HTTPs request to the server
         url_data = {"database" : self.data_to_send["destinationDatabase"], "upload_session_id" : upload_session_id, "tier0_response" : self.data_to_send.get("tier0_response")}
         request = url_query(url=self._SERVICE_URL + "upload_metadata/", url_data=url_data, body=json.dumps(self.data_to_send))
         response = request.send()
         self._outputter.write("Response received - conditions upload process complete.")
         return response
 
 if __name__ == "__main__":
     """
     This code should only be executed for testing.
     """
     import sys
     from .uploadConditions import parse_arguments
 
     print(
 """
 This code should only be executed for testing.
 Any uploads done by the user should be done by calling the uploadConditions.py script.
 See https://cms-conddb-dev.cern.ch/cmsDbCondUpload for information on how to obtain the correct version.
 """
     )
 
     upload_metadata = parse_arguments()
 
     upload_metadata["sqlite_file"] = upload_metadata.get("sourceDB")
 
     # make new dictionary, and copy over everything except "metadata_source"
     upload_metadata_argument = {}
     for (key, value) in list(upload_metadata.items()):
         if key != "metadata_source":
             upload_metadata_argument[key] = value
 
     upload_metadata["metadata_source"] = data_sources.json_data_node.make(upload_metadata_argument)
 
     upload_controller = uploader(**upload_metadata)
 
     result = upload_controller.upload()

This page was automatically generated by the 2.2.1 LXR engine. The LXR team