python/CondDBFW/models.py

0001 """
0002
0003 Using Audrius' models from flask browser.
0004
0005 This file contains models that are used with SQLAlchemy.
0006
0007 Note: some things done in methods written in classes rely on the querying module adding extra information to classes,
0008       so these will not work in a normal context outside the framework.
0009
0010 """
0011 import json
0012 import datetime
0013
0014 try:
0015     import sqlalchemy
0016     from sqlalchemy.orm import relationship, backref
0017     from sqlalchemy.ext.declarative import declarative_base
0018     # Note: Binary is only used for blobs, if they are mapped
0019     from sqlalchemy import Column, String, Integer, DateTime, Binary, ForeignKey, BigInteger, and_
0020 except ImportError:
0021     print("You must be working inside a CMSSW environment.  Try running 'cmsenv'.")
0022     exit()
0023
0024 from . import data_sources, data_formats
0025 import urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse, base64
0026 from copy import deepcopy
0027
0028 # get utility functions
0029 from .utils import to_timestamp, to_datetime, friendly_since
0030
0031 def session_independent_object(object, schema=None):
0032     # code original taken from write method in querying
0033     # will result in a new object that isn't attached to any session
0034     # hence, SQLAlchemy won't track changes
0035
0036     if object.__class__.__name__.lower() == "payload":
0037         map_blobs = object.blobs_mapped
0038     else:
0039         map_blobs = False
0040     # need to change this to only generate the required class - can be slow...
0041     # extract class name of object
0042     cls = object.__class__
0043     class_name = class_name_to_column(cls).lower()
0044     new_class = generate(map_blobs=map_blobs, class_name=class_name)
0045     new_class.__table__.schema = schema
0046     new_object = new_class(object.as_dicts(), convert_timestamps=False)
0047
0048     return new_object
0049
0050 def session_independent(objects):
0051     if type(objects) == list:
0052         return list(map(session_independent_object, objects))
0053     else:
0054         # assume objects is a single object (not a list)
0055         return session_independent_object(objects)
0056
0057 def class_name_to_column(cls):
0058     class_name = cls.__name__
0059     all_upper_case = True
0060     for character in class_name:
0061         all_upper_case = character.isupper()
0062     if all_upper_case:
0063         return class_name
0064     for n in range(0, len(class_name)):
0065         if class_name[n].isupper() and n != 0:
0066             class_name = str(class_name[0:n]) + "".join(["_", class_name[n].lower()]) + str(class_name[n+1:])
0067         elif class_name[n].isupper() and n == 0:
0068             class_name = str(class_name[0:n]) + "".join([class_name[n].lower()]) + str(class_name[n+1:])
0069     return class_name
0070
0071 def status_full_name(status):
0072     full_status = {
0073         'P': 'Pending',
0074         'R': 'Rejected',
0075         'A': 'Accepted'
0076     }
0077     return full_status[status]
0078
0079 def date_args_to_days(**radius):
0080     days = radius.get("days")
0081     days += radius.get("weeks")*7 if radius.get("weeks") != None else 0
0082     days += radius.get("months")*28 if radius.get("months") != None else 0
0083     days += radius.get("years")+365 if radius.get("years") != None else 0
0084     return days
0085
0086 class ContinuousRange(object):
0087     """
0088     Base class for Radius and Range - used for checking by apply_filter function
0089     """
0090
0091     def __init__(self):
0092         pass
0093
0094     def get_start(self):
0095         return self._start
0096
0097     def get_end(self):
0098         return self._end
0099
0100 class Radius(ContinuousRange):
0101     """
0102     Used to tell proxy methods that a range of values defined by a centre and a radius should be queried for - special case of filter clauses.
0103     """
0104     def __init__(self, centre, radius):
0105         """
0106         centre and radius should be objects that can be added and subtracted.
0107         eg, centre could be a datetime.datetime object, and radius could be datetime.timedelta
0108
0109         Radius and Range objects are assigned to properties of querying.connection objects, hence are given the database type.
0110         """
0111         self._centre = centre
0112         self._radius = radius
0113         self._start = self._centre - self._radius
0114         self._end = self._centre + self._radius
0115
0116 class Range(ContinuousRange):
0117     """
0118     Used to tell proxy methods that a range of values defined by a start and end point should be queried for - special case of filter clauses.
0119     """
0120     def __init__(self, start, end):
0121         """
0122         centre and radius should be objects that can be added and subtracted.
0123         eg, centre could be a datetime.datetime object, and radius could be datetime.timedelta
0124
0125         Radius and Range objects are assigned to properties of querying.connection objects, hence are given the database type.
0126         """
0127         self._start = start
0128         self._end = end
0129
0130 class RegExp(object):
0131     """
0132     Used to tell proxy methods that a regular expression should be used to query the column.
0133     """
0134     def __init__(self, regexp):
0135         self._regexp = regexp
0136
0137     def get_regexp(self):
0138         return self._regexp
0139
0140     def apply(self):
0141         # uses code from conddb tool
0142         if self.database_type in ["oracle", "frontier"]:
0143             return sqlalchemy.func.regexp_like(field, regexp)
0144         elif self.database_type == "sqlite":
0145             # Relies on being a SingletonThreadPool
0146             self.connection_object.engine.pool.connect().create_function('regexp', 2, lambda data, regexp: re.search(regexp, data) is not None)
0147
0148             return sqlalchemy.func.regexp(field, regexp)
0149         else:
0150             raise NotImplemented("Can only apply regular expression search to Oracle, Frontier and SQLite.")
0151
0152 def apply_filter(orm_query, orm_class, attribute, value):
0153     filter_attribute = getattr(orm_class, attribute)
0154     if type(value) == list:
0155         orm_query = orm_query.filter(filter_attribute.in_(value))
0156     elif type(value) == data_sources.json_list:
0157         orm_query = orm_query.filter(filter_attribute.in_(value.data()))
0158     elif type(value) in [Range, Radius]:
0159
0160         minus = value.get_start()
0161         plus = value.get_end()
0162         orm_query = orm_query.filter(and_(filter_attribute >= minus, filter_attribute <= plus))
0163
0164     elif type(value) == RegExp:
0165
0166         # Relies on being a SingletonThreadPool
0167
0168         if value.database_type in ["oracle", "frontier"]:
0169             regexp = sqlalchemy.func.regexp_like(filter_attribute, value.get_regexp())
0170         elif value.database_type == "sqlite":
0171             value.connection_object.engine.pool.connect().create_function('regexp', 2, lambda data, regexp: re.search(regexp, data) is not None)
0172             regexp = sqlalchemy.func.regexp(filter_attribute, value.get_regexp())
0173         else:
0174             raise NotImplemented("Can only apply regular expression search to Oracle, Frontier and SQLite.")
0175         orm_query = orm_query.filter(regexp)
0176
0177     else:
0178         orm_query = orm_query.filter(filter_attribute == value)
0179     return orm_query
0180
0181 def apply_filters(orm_query, orm_class, **filters):
0182     for (key, value) in list(filters.items()):
0183         if not(key in ["amount"]):
0184             orm_query = apply_filter(orm_query, orm_class, key, value)
0185     return orm_query
0186
0187 def generate(map_blobs=False, class_name=None):
0188
0189     Base = declarative_base()
0190     schema = {"schema" : "CMS_CONDITIONS"}
0191     fk_schema_prefix = ("%s." % schema["schema"]) if schema else ""
0192
0193     class GlobalTag(Base):
0194         __table_args__ = schema
0195         __tablename__ = 'GLOBAL_TAG'
0196
0197         headers = ["name", "validity", "description", "release", "insertion_time", "snapshot_time", "scenario", "workflow", "type"]
0198
0199         name = Column(String(100), unique=True, nullable=False, primary_key=True)
0200         validity = Column(Integer, nullable=False)
0201         description = Column(String(4000), nullable=False)
0202         release = Column(String(100), nullable=False)
0203         insertion_time = Column(DateTime, nullable=False)
0204         snapshot_time = Column(DateTime, nullable=False)
0205         scenario = Column(String(100))
0206         workflow = Column(String(100))
0207         type = Column(String(1))
0208         tag_map = relationship('GlobalTagMap', backref='global_tag')
0209
0210         def __init__(self, dictionary={}, convert_timestamps=True):
0211             # assign each entry in a kwargs
0212             for key in dictionary:
0213                 try:
0214                     if convert_timestamps:
0215                         self.__dict__[key] = to_timestamp(dictionary[key])
0216                     else:
0217                         self.__dict__[key] = dictionary[key]
0218                 except KeyError as k:
0219                     continue
0220
0221         def __repr__(self):
0222             return '<GlobalTag %r>' % self.name
0223
0224         def as_dicts(self, convert_timestamps=False):
0225             """
0226             Returns dictionary form of Global Tag object.
0227             """
0228             json_gt = {
0229                 'name': self.name,
0230                 'validity': self.validity,
0231                 'description': self.description,
0232                 'release': self.release,
0233                 'insertion_time': to_timestamp(self.insertion_time) if convert_timestamps else self.insertion_time,
0234                 'snapshot_time': to_timestamp(self.snapshot_time) if convert_timestamps else self.snapshot_time,
0235                 'scenario': self.scenario,
0236                 'workflow': self.workflow,
0237                 'type': self.type
0238             }
0239             return json_gt
0240
0241         def to_array(self):
0242             return [self.name, self.release, to_timestamp(self.insertion_time), to_timestamp(self.snapshot_time), self.description]
0243
0244         def all(self, **kwargs):
0245             """
0246             Returns `amount` Global Tags ordered by Global Tag name.
0247             """
0248             query = self.session.query(GlobalTag)
0249             query = apply_filters(query, self.__class__, **kwargs)
0250             amount = kwargs["amount"] if "amount" in list(kwargs.keys()) else None
0251             query_result = query.order_by(GlobalTag.name).limit(amount).all()
0252             gts = data_sources.json_data_node.make(query_result)
0253             return gts
0254
0255         def tags(self, **kwargs):
0256             """
0257             Returns `amount` *Global Tag Maps* belonging to this Global Tag.
0258             """
0259             kwargs["global_tag_name"] = self.name
0260             all_tags = self.session.query(GlobalTagMap.global_tag_name, GlobalTagMap.record, GlobalTagMap.label, GlobalTagMap.tag_name)
0261             all_tags = apply_filters(all_tags, GlobalTagMap, **kwargs)
0262             amount = kwargs["amount"] if "amount" in list(kwargs.keys()) else None
0263             all_tags = all_tags.order_by(GlobalTagMap.tag_name).limit(amount).all()
0264             column_names = ["global_tag_name", "record", "label", "tag_name"]
0265             all_tags = [dict(list(zip(column_names, list(map(to_timestamp, row))))) for row in all_tags]
0266             all_tags = data_formats._dicts_to_orm_objects(GlobalTagMap, all_tags)
0267             return data_sources.json_data_node.make(all_tags)
0268
0269         def iovs(self, **kwargs):
0270             """
0271             Returns `amount` IOVs belonging to all Tags held in this Global Tag.
0272             For large Global Tags (which is most of them), VERY slow.
0273             Highly recommended to instead used `tags().get_members("tag_name").data()` to get a `list` of tag names,
0274             and then get IOVs from each Tag name.
0275
0276             At some point, this method may replace the method currently used.
0277             """
0278             # join global_tag_map onto iov (where insertion time <= gt snapshot) by tag_name + return results
0279             # first get only the IOVs that belong to Tags that are contained by this Global Tag
0280
0281             # get IOVs belonging to a Tag contained by this Global Tag
0282             tag_names = self.tags().get_members("tag_name").data()
0283             iovs_all_tags = self.session.query(IOV).filter(IOV.tag_name.in_(tag_names))
0284             iovs_all_tags = apply_filters(iovs_all_tags, IOV, **kwargs)
0285             amount = kwargs["amount"] if "amount" in list(kwargs.keys()) else None
0286             iovs_all_tags = iovs_all_tags.limit(amount).subquery()
0287
0288             # now, join Global Tag Map table onto IOVs
0289             iovs_gt_tags = self.session.query(GlobalTagMap.tag_name, iovs_all_tags.c.since,\
0290                                                     iovs_all_tags.c.payload_hash, iovs_all_tags.c.insertion_time)\
0291                                             .filter(GlobalTagMap.global_tag_name == self.name)\
0292                                             .join(iovs_all_tags, GlobalTagMap.tag_name == iovs_all_tags.c.tag_name)
0293
0294             iovs_gt_tags = iovs_gt_tags.order_by(iovs_all_tags.c.since).all()
0295
0296             column_names = ["tag_name", "since", "payload_hash", "insertion_time"]
0297             all_iovs = [dict(list(zip(column_names, row))) for row in iovs_gt_tags]
0298             all_iovs = data_formats._dicts_to_orm_objects(IOV, all_iovs)
0299
0300             return data_sources.json_data_node.make(all_iovs)
0301
0302         def __sub__(self, other):
0303             """
0304             Allows Global Tag objects to be used with the "-" arithmetic operator to find their difference.
0305             Note: gt1 - gt2 = gt1.diff(gt2) ( = gt2 - gt1 = gt2.diff(gt1))
0306             """
0307             return self.diff(other)
0308
0309         def diff(self, gt):
0310             """
0311             Returns the json_list of differences in the form of tuples:
0312
0313             (record, label, tag name of gt1 (self), tag name of gt2 (gt))
0314             """
0315
0316             record_label_to_tag_name1 = dict([((gt_map.record, gt_map.label), gt_map.tag_name) for gt_map in self.tags().data()])
0317             record_label_to_tag_name2 = dict([((gt_map.record, gt_map.label), gt_map.tag_name) for gt_map in gt.tags().data()])
0318
0319             record_label_pairs = sorted(set(record_label_to_tag_name1) | set(record_label_to_tag_name2))
0320
0321             table = []
0322             tags_pairs_with_differences = []
0323
0324             for record_label in record_label_pairs:
0325                 tag_name1 = record_label_to_tag_name1.get(record_label)
0326                 tag_name2 = record_label_to_tag_name2.get(record_label)
0327
0328                 if tag_name1 == None or tag_name2 == None or tag_name1 != tag_name2:
0329                     table.append({
0330                             "Record" : record_label[0],
0331                             "Label" : record_label[1],
0332                             ("%s Tag" % self.name) : tag_name1,
0333                             ("%s Tag" % gt.name) : tag_name2
0334                         })
0335
0336             return data_sources.json_data_node.make(table)
0337
0338     class GlobalTagMap(Base):
0339         __table_args__ = schema
0340         __tablename__ = 'GLOBAL_TAG_MAP'
0341
0342         headers = ["global_tag_name", "record", "label", "tag_name"]
0343
0344         global_tag_name = Column(String(100), ForeignKey(fk_schema_prefix + 'GLOBAL_TAG.name'), primary_key=True, nullable=False)
0345         record = Column(String(100), ForeignKey(fk_schema_prefix + 'RECORDS.record'), primary_key=True, nullable=False)
0346         label = Column(String(100), primary_key=True, nullable=False)
0347         tag_name = Column(String(100), ForeignKey(fk_schema_prefix + 'TAG.name'), nullable=False)
0348
0349         def __init__(self, dictionary={}, convert_timestamps=True):
0350             # assign each entry in a kwargs
0351             for key in dictionary:
0352                 try:
0353                     if convert_timestamps:
0354                         self.__dict__[key] = to_timestamp(dictionary[key])
0355                     else:
0356                         self.__dict__[key] = dictionary[key]
0357                 except KeyError as k:
0358                     continue
0359
0360         def __repr__(self):
0361             return '<GlobalTagMap %r>' % self.global_tag_name
0362
0363         def as_dicts(self, convert_timestamps=False):
0364             """
0365             Returns dictionary form of this Global Tag Map.
0366             """
0367             json_gtm = {
0368                 "global_tag_name" : str(self.global_tag_name),
0369                 "record" : str(self.record),
0370                 "label" : str(self.label),
0371                 "tag_name" : str(self.tag_name)
0372             }
0373             return json_gtm
0374
0375
0376     class GlobalTagMapRequest(Base):
0377         __table_args__ = schema
0378         __tablename__ = 'GLOBAL_TAG_MAP_REQUEST'
0379
0380         queue = Column(String(100), primary_key=True, nullable=False)
0381         tag = Column(String(100), ForeignKey(fk_schema_prefix + 'TAG.name'), primary_key=True, nullable=False)
0382         record = Column(String(100), ForeignKey(fk_schema_prefix + 'RECORDS.record'), primary_key=True, nullable=False)
0383         label = Column(String(100), primary_key=True, nullable=False)
0384         status = Column(String(1), nullable=False)
0385         description = Column(String(4000), nullable=False)
0386         submitter_id = Column(Integer, nullable=False)
0387         time_submitted = Column(DateTime, nullable=False)
0388         last_edited = Column(DateTime, nullable=False)
0389
0390         def __init__(self, dictionary={}, convert_timestamps=True):
0391             # assign each entry in a kwargs
0392             for key in dictionary:
0393                 try:
0394                     if convert_timestamps:
0395                         self.__dict__[key] = to_timestamp(dictionary[key])
0396                     else:
0397                         self.__dict__[key] = dictionary[key]
0398                 except KeyError as k:
0399                     continue
0400
0401         headers = ["queue", "tag", "record", "label", "status", "description", "submitter_id", "time_submitted", "last_edited"]
0402
0403         def as_dicts(self):
0404             """
0405             Returns dictionary form of this Global Tag Map Request.
0406             """
0407             return {
0408                 "queue" : self.queue,
0409                 "tag" : self.tag,
0410                 "record" : self.record,
0411                 "label" : self.label,
0412                 "status" : self.status,
0413                 "description" : self.description,
0414                 "submitter_id" : self.submitter_id,
0415                 "time_submitted" : self.time_submitted,
0416                 "last_edited" : self.last_edited
0417             }
0418
0419         def __repr__(self):
0420             return '<GlobalTagMapRequest %r>' % self.queue
0421
0422         def to_array(self):
0423             return [self.queue, self.tag, self.record, self.label, status_full_name(self.status), to_timestamp(self.time_submitted), to_timestamp(self.last_edited)]
0424
0425     class IOV(Base):
0426         __table_args__ = schema
0427         __tablename__ = 'IOV'
0428
0429         headers = ["tag_name", "since", "payload_hash", "insertion_time"]
0430
0431         tag_name = Column(String(4000), ForeignKey(fk_schema_prefix + 'TAG.name'), primary_key=True, nullable=False)
0432         since = Column(Integer, primary_key=True, nullable=False)
0433         payload_hash = Column(String(40), ForeignKey(fk_schema_prefix + 'PAYLOAD.hash'), nullable=False)
0434         insertion_time = Column(DateTime, primary_key=True, nullable=False)
0435
0436         def __init__(self, dictionary={}, convert_timestamps=True):
0437             # assign each entry in a kwargs
0438             for key in dictionary:
0439                 try:
0440                     if convert_timestamps:
0441                         self.__dict__[key] = to_timestamp(dictionary[key])
0442                     else:
0443                         self.__dict__[key] = dictionary[key]
0444                 except KeyError as k:
0445                     continue
0446
0447         def as_dicts(self, convert_timestamps=False):
0448             """
0449             Returns dictionary form of this IOV.
0450             """
0451             return {
0452                 "tag_name" : self.tag_name,
0453                 "since" : self.since,
0454                 "payload_hash" : self.payload_hash,
0455                 "insertion_time" : to_timestamp(self.insertion_time) if convert_timestamps else self.insertion_time
0456             }
0457
0458         def __repr__(self):
0459             return '<IOV %r>' % self.tag_name
0460
0461         def to_array(self):
0462             return [self.since, to_timestamp(self.insertion_time), self.payload_hash]
0463
0464         def all(self, **kwargs):
0465             """
0466             Returns `amount` IOVs ordered by since.
0467             """
0468             query = self.session.query(IOV)
0469             query = apply_filters(query, IOV, **kwargs)
0470             amount = kwargs["amount"] if "amount" in list(kwargs.keys()) else None
0471             query_result = query.order_by(IOV.tag_name).order_by(IOV.since).limit(amount).all()
0472             return data_sources.json_data_node.make(query_result)
0473
0474
0475     class Payload(Base):
0476         __table_args__ = schema
0477         __tablename__ = 'PAYLOAD'
0478
0479         headers = ["hash", "object_type", "version", "insertion_time"]
0480
0481         hash = Column(String(40), primary_key=True, nullable=False)
0482         object_type = Column(String(4000), nullable=False)
0483         version = Column(String(4000), nullable=False)
0484         insertion_time = Column(DateTime, nullable=False)
0485         if map_blobs:
0486             data = Column(Binary, nullable=False)
0487             streamer_info = Column(Binary, nullable=False)
0488         blobs_mapped = map_blobs
0489
0490         def __init__(self, dictionary={}, convert_timestamps=True):
0491             # assign each entry in a kwargs
0492             for key in dictionary:
0493                 try:
0494                     if convert_timestamps:
0495                         self.__dict__[key] = to_timestamp(dictionary[key])
0496                     else:
0497                         self.__dict__[key] = dictionary[key]
0498                 except KeyError as k:
0499                     continue
0500
0501         if map_blobs:
0502             def as_dicts(self, convert_timestamps=False):
0503                 """
0504                 Returns dictionary form of this Payload's metadata (not the actual Payload).
0505                 """
0506                 return {
0507                     "hash" : self.hash,
0508                     "object_type" : self.object_type,
0509                     "version" : self.version,
0510                     "insertion_time" : to_timestamp(self.insertion_time) if convert_timestamps else self.insertion_time,
0511                     "data" : self.data,
0512                     "streamer_info" : self.streamer_info
0513                 }
0514         else:
0515             def as_dicts(self, convert_timestamps=False):
0516                 """
0517                 Returns dictionary form of this Payload's metadata (not the actual Payload).
0518                 """
0519                 return {
0520                     "hash" : self.hash,
0521                     "object_type" : self.object_type,
0522                     "version" : self.version,
0523                     "insertion_time" : to_timestamp(self.insertion_time) if convert_timestamps else self.insertion_time
0524                 }
0525
0526         def __repr__(self):
0527             return '<Payload %r>' % self.hash
0528
0529         def to_array(self):
0530             return [self.hash, self.object_type, self.version, to_timestamp(self.insertion_time)]
0531
0532         def parent_tags(self, **kwargs):
0533             """
0534             Returns `amount` parent Tags ordered by Tag name.
0535             """
0536             # check if this payload is empty
0537             if self.empty:
0538                 return None
0539             else:
0540                 kwargs["payload_hash"] = self.hash
0541                 query = self.session.query(IOV.tag_name)
0542                 query = apply_filters(query, IOV, **kwargs)
0543                 query_result = query.all()
0544                 tag_names = [entry[0] for entry in query_result]
0545                 amount = kwargs["amount"] if "amount" in list(kwargs.keys()) else None
0546                 tags = self.session.query(Tag).filter(Tag.name.in_(tag_names)).order_by(Tag.name).limit(amount).all()
0547                 return data_sources.json_data_node.make(tags)
0548
0549         def all(self, **kwargs):
0550             """
0551             Returns `amount` Payloads ordered by Payload hash.
0552             """
0553             query = self.session.query(Payload)
0554             query = apply_filters(query, Payload, **kwargs)
0555             amount = kwargs["amount"] if "amount" in list(kwargs.keys()) else None
0556             query_result = query.order_by(Payload.hash).limit(amount).all()
0557             return data_sources.json_data_node.make(query_result)
0558
0559
0560     class Record(Base):
0561         __table_args__ = schema
0562         __tablename__ = 'RECORDS'
0563
0564         headers = ["record", "object", "type"]
0565
0566         record = Column(String(100), primary_key=True, nullable=False)
0567         object = Column(String(200), nullable=False)
0568         type = Column(String(20), nullable=False)
0569
0570         def as_dicts(self):
0571             """
0572             Returns dictionary form of this Record.
0573             """
0574             return {
0575                 "record" : self.record,
0576                 "object" : self.object,
0577                 "type" : self.type
0578             }
0579
0580         def __repr__(self):
0581             return '<Record %r>' % self.record
0582
0583         def to_array(self):
0584             return [self.record, self.object]
0585
0586         def all(self, **kwargs):
0587             """
0588             Returns `amount` Records ordered by Record record.
0589             """
0590             query = self.session.query(Record)
0591             query = apply_filters(query, Record, kwargs)
0592             amount = kwargs["amount"] if "amount" in list(kwargs.keys()) else None
0593             query_result = query.order_by(Record.record).limit(amount).all()
0594             return data_sources.json_data_node.make(query_result)
0595
0596
0597     class Tag(Base):
0598         __table_args__ = schema
0599         __tablename__ = 'TAG'
0600
0601         headers = ["name", "time_type", "object_type", "synchronization", "end_of_validity",\
0602                     "description", "last_validated_time", "insertion_time", "modification_time", "protection_code"]
0603
0604         name = Column(String(4000), primary_key=True, nullable=False)
0605         time_type = Column(String(4000), nullable=False)
0606         object_type = Column(String(4000), nullable=False)
0607         synchronization = Column(String(4000), nullable=False)
0608         end_of_validity = Column(Integer, nullable=False)
0609         description = Column(String(4000), nullable=False)
0610         last_validated_time = Column(BigInteger, nullable=False)
0611         insertion_time = Column(DateTime, nullable=False)
0612         modification_time = Column(DateTime, nullable=False)
0613         protection_code = Column(Integer, nullable=False)
0614
0615         record = None
0616         label = None
0617
0618         iovs_list = relationship('IOV', backref='tag')
0619
0620         def __init__(self, dictionary={}, convert_timestamps=True):
0621             # assign each entry in a kwargs
0622             for key in dictionary:
0623                 try:
0624                     if convert_timestamps:
0625                         self.__dict__[key] = to_timestamp(dictionary[key])
0626                     else:
0627                         self.__dict__[key] = dictionary[key]
0628                 except KeyError as k:
0629                     continue
0630
0631         def as_dicts(self, convert_timestamps=False):
0632             """
0633             Returns dictionary form of this Tag.
0634             """
0635             return {
0636                 "name" : self.name,
0637                 "time_type" : self.time_type,
0638                 "object_type" : self.object_type,
0639                 "synchronization" : self.synchronization,
0640                 "end_of_validity" : self.end_of_validity,
0641                 "description" : self.description,
0642                 "last_validated_time" : self.last_validated_time,
0643                 "insertion_time" : to_timestamp(self.insertion_time) if convert_timestamps else self.insertion_time,
0644                 "modification_time" : to_timestamp(self.modification_time) if convert_timestamps else self.modification_time,
0645                 "record" : self.record,
0646                 "label" : self.label
0647             }
0648
0649         def __repr__(self):
0650             return '<Tag %r>' % self.name
0651
0652         def to_array(self):
0653             return [self.name, self.time_type, self.object_type, self.synchronization, to_timestamp(self.insertion_time), self.description]
0654
0655         def parent_global_tags(self, **kwargs):
0656             """
0657             Returns `amount` Global Tags that contain this Tag.
0658             """
0659             if self.empty:
0660                 return None
0661             else:
0662                 kwargs["tag_name"] = self.name
0663                 query = self.session.query(GlobalTagMap.global_tag_name)
0664                 query = apply_filters(query, GlobalTagMap, **kwargs)
0665                 query_result = query.all()
0666                 if len(query_result) != 0:
0667                     global_tag_names = [entry[0] for entry in query_result]
0668                     amount = kwargs["amount"] if "amount" in list(kwargs.keys()) else None
0669                     global_tags = self.session.query(GlobalTag).filter(GlobalTag.name.in_(global_tag_names)).order_by(GlobalTag.name).limit(amount).all()
0670                 else:
0671                     global_tags = None
0672                 return data_sources.json_data_node.make(global_tags)
0673
0674         def all(self, **kwargs):
0675             """
0676             Returns `amount` Tags ordered by Tag name.
0677             """
0678             query = self.session.query(Tag)
0679             query = apply_filters(query, Tag, **kwargs)
0680             amount = kwargs["amount"] if "amount" in list(kwargs.keys()) else None
0681             query_result = query.order_by(Tag.name).limit(amount).all()
0682             return data_sources.json_data_node.make(query_result)
0683
0684         def iovs(self, **kwargs):
0685             """
0686             Returns `amount` IOVs that belong to this Tag ordered by IOV since.
0687             """
0688             # filter_params contains a list of columns to filter the iovs by
0689             iovs_query = self.session.query(IOV).filter(IOV.tag_name == self.name)
0690             iovs_query = apply_filters(iovs_query, IOV, **kwargs)
0691             amount = kwargs["amount"] if "amount" in list(kwargs.keys()) else None
0692             iovs = iovs_query.order_by(IOV.since).limit(amount).all()
0693             return data_sources.json_data_node.make(iovs)
0694
0695         def latest_iov(self):
0696             """
0697             Returns the single highest since held by this Tag.
0698             Insertion times do not matter - if there are two IOVs at since > all others, both have the highest since.
0699             """
0700             iov = self.session.query(IOV).filter(IOV.tag_name == self.name).order_by(IOV.since.desc()).first()
0701             return iov
0702
0703         def __sub__(self, other):
0704             """
0705             Allows the arithmetic operator "-" to be applied to find the difference between two tags.
0706             Note: diff() is symmetric, hence tag1 - tag2 = tag2 - tag1.
0707             """
0708             return self.diff(other)
0709
0710         def diff(self, tag, short=False):
0711             """
0712             Returns the `diff` of the first Tag, and the Tag given.
0713             Summary of algorithm:
0714
0715             Compute the ordered set of iov sinces from both tags, and construct a list of triples, (since, tag1 hash, tag2 hash).
0716             Set previous_payload1 and previous_payload2 to be the first hash values from each tag for the first since in the merged list.
0717                 Note: depending on where each Tag's IOVs start, 1 or both of these values can be None.
0718             Set the first_since_in_equality_range = -1, which holds the since at which the last hashes were equal in the Tags.
0719             For each triple (since, hash1, hash2),
0720
0721                 If the first_since_in_equality_range = None,
0722                     We are at the first since in the merged list, so set first_since... = since
0723                     Note: this is so set the previous... values for the second row, since the first row will never result in a print because
0724                     a row is only printed when past iovs have been processed.
0725
0726                 If either hash1 or hash2 is None, set it to the previous hash found
0727                     Note: if a Tag defines a hash for one since and then not another for n rows, the last defined hash will be carried through because of this.
0728
0729                 If the previous found hashes were equal, that means we have equality on the range [first_since_in_equality_range, since)
0730                     Note: we CANNOT conclude anything about the hashes corresponding to sinces >= since
0731                             because we have no looked forward, but we do know about the previous hashes.
0732
0733                     If hash1 != hash2,
0734                         The region of equality has ended, and so we have that [first_since_in_equality_range, since) is equal for both Tags
0735                         Hence, print that for this range we have equal hashes denoted by "=" in each hash column.
0736
0737                 Else:
0738
0739                     The previous hashes were not equal, BUT we must check that ths hashes on this row are not identical...
0740                     If the hashes on this row are the same as the hashes above (hash1 == previous_payload1 and hash2 == previous_payload2),
0741                     then we have not found the end of a region of equality!
0742                     If the hashes have changed, print a row.
0743
0744             """
0745             if tag.__class__.__name__ != "Tag":
0746                 raise TypeError("Tag given must be a CondDBFW Tag object.")
0747
0748             # get lists of iovs
0749             iovs1 = dict([(iov.since, iov.payload_hash) for iov in self.iovs().data()])
0750             iovs2 = dict([(iov.since, iov.payload_hash) for iov in tag.iovs().data()])
0751
0752             iovs = [(x, iovs1.get(x), iovs2.get(x)) for x in sorted(set(iovs1) | set(iovs2))]
0753             iovs.append(("Infinity", 1, 2))
0754             table = []
0755
0756             previous_hash1 = None
0757             previous_hash2 = None
0758             first_since_in_equality_range = None
0759             previous_equal = False
0760
0761             for since, hash1, hash2 in iovs:
0762
0763                 if first_since_in_equality_range == None:
0764                     # if no start of a region of equality has been found,
0765                     # set it to the first since in the merged list
0766                     # then set the previous hashes and equality status to the current
0767                     # and continue to the next iteration of the loop
0768                     first_since_in_equality_range = since
0769                     previous_hash1 = hash1
0770                     previous_hash2 = hash2
0771                     previous_equal = hash1 == hash2
0772                     continue
0773
0774                 # if previous_payload1 is also None, comparisons still matters
0775                 # eg, if hash1 = None and hash2 != None, they are different and so should be shown in the table
0776                 if hash1 == None:
0777                     hash1 = previous_hash1
0778                 if hash2 == None:
0779                     hash2 = previous_hash2
0780
0781                 if previous_equal:
0782                     # previous hashes were equal, but only say they were if we have found an end of the region of equality
0783                     if hash1 != hash2:
0784                         table.append({"since" : "[%s, %s)" % (first_since_in_equality_range, since), self.name : "=", tag.name : "="})
0785                         # this is the start of a new equality range - might only be one row if the next row has unequal hashes!
0786                         first_since_in_equality_range = since
0787                 else:
0788                     # if the payloads are not equal, the equality range has ended and we should print a row
0789                     # we only print if EITHER hash has changed
0790                     # if both hashes are equal to the previous row, skip to the next row to try to find the beginning
0791                     # of a region of equality
0792                     if not(hash1 == previous_hash1 and hash2 == previous_hash2):
0793                         table.append({"since" : "[%s, %s)" % (first_since_in_equality_range, since), self.name : previous_hash1, tag.name : previous_hash2})
0794                         first_since_in_equality_range = since
0795
0796                 previous_hash1 = hash1
0797                 previous_hash2 = hash2
0798                 previous_equal = hash1 == hash2
0799
0800             final_list = data_sources.json_data_node.make(table)
0801             return final_list
0802
0803         def merge_into(self, tag, range_object):
0804             """
0805             Given another connection, apply the 'merge' algorithm to merge the IOVs from this Tag
0806             into the IOVs of the other Tag.
0807
0808             tag : CondDBFW Tag object that the IOVs from this Tag should be merged into.
0809
0810             range_object : CondDBFW.data_sources.Range object to describe the subset of IOVs that should be copied
0811             from the database this Tag belongs to.
0812
0813             Script originally written by Joshua Dawes,
0814             and adapted by Giacomo Govi, Gianluca Cerminara and Giovanni Franzoni.
0815             """
0816
0817             oracle_tag = self
0818             merged_tag_name = oracle_tag.name + "_merged"
0819
0820             #since_range = Range(6285191841738391552,6286157702573850624)
0821             since_range = range_object
0822
0823             #sqlite = shell.connect("sqlite://EcallaserTag_80X_2016_prompt_corr20160519_2.db")
0824
0825             #sqlite_tag = sqlite.tag().all().data()[0]
0826             sqlite_tag = tag
0827             if sqlite_tag == None:
0828                 raise TypeError("Tag to be merged cannot be None.")
0829
0830             sqlite_iovs = sqlite_tag.iovs().data()
0831             sqlite_tag.iovs().as_table()
0832
0833             new_tag = self.connection.models["tag"](sqlite_tag.as_dicts(convert_timestamps=False), convert_timestamps=False)
0834             new_tag.name = merged_tag_name
0835
0836             imported_iovs = oracle_tag.iovs(since=since_range).data()
0837
0838             for i in range(0, len(imported_iovs)):
0839                 imported_iovs[i].source = "oracle"
0840
0841             sqlite_iovs_sinces=[]
0842             for i in range(0, len(sqlite_iovs)):
0843                 sqlite_iovs[i].source = "sqlite"
0844                 sqlite_iovs_sinces.append(sqlite_iovs[i].since)
0845
0846
0847             print(sqlite_iovs_sinces)
0848
0849             new_iovs_list = imported_iovs + sqlite_iovs
0850             new_iovs_list = sorted(new_iovs_list, key=lambda iov : iov.since)
0851
0852             for (n, iov) in enumerate(new_iovs_list):
0853                 # if iov is from oracle, change its hash
0854                 if iov.source == "oracle":
0855                     if new_iovs_list[n].since in sqlite_iovs_sinces:
0856                         # if its since is already defined in the target iovs
0857                         # ignore it
0858                         iov.source = "tobedeleted"
0859                     else:
0860                         # otherwise, iterate down from n to find the last sqlite iov,
0861                         # and assign that hash
0862                         for i in reversed(list(range(0,n))):
0863                             if new_iovs_list[i].source == "sqlite":
0864                                 print("change %s to %s at since %d" % (iov.payload_hash, new_iovs_list[i].payload_hash, iov.since))
0865                                 iov.payload_hash = new_iovs_list[i].payload_hash
0866                                 break
0867
0868
0869             new_iov_list_copied = []
0870
0871             for iov in new_iovs_list:
0872                 # only append IOVs that are not already defined in the target tag
0873                 if iov.source != "tobedeleted":
0874                     new_iov_list_copied.append(iov)
0875
0876             new_iov_list_copied = sorted(new_iov_list_copied, key=lambda iov : iov.since)
0877
0878             now = datetime.datetime.utcnow()
0879
0880             new_iovs = []
0881             for iov in new_iov_list_copied:
0882                 new_iovs.append( self.connection.models["iov"](iov.as_dicts(convert_timestamps=False), convert_timestamps=False)  )
0883             for iov in new_iovs:
0884                 iov.insertion_time = now
0885                 iov.tag_name = merged_tag_name
0886
0887             new_tag.iovs_list = new_iovs
0888
0889             return new_tag
0890             #sqlite.write_and_commit(new_iovs)
0891
0892
0893     class TagAuthorization(Base):
0894         __table_args__ = schema
0895         __tablename__ = 'TAG_AUTHORIZATION'
0896
0897         headers = ["tag_name", "access_type", "credential", "credential_type"]
0898
0899         tag_name = Column(String(100), ForeignKey(fk_schema_prefix + 'TAG.name'), primary_key=True, nullable=False)
0900         access_type = Column(Integer, nullable=False)
0901         credential = Column(String(100), primary_key=True, nullable=False)
0902         credential_type = Column(Integer, nullable=False)
0903
0904         def as_dicts(self):
0905             """
0906             Returns dictionary form of this Tag Authorization.
0907             """
0908             return {
0909                 "tag_name" : self.tag_name,
0910                 "access_type" : self.access_type,
0911                 "credential" : self.credential,
0912                 "credential_type" : self.credential_type
0913             }
0914
0915         def __repr__(self):
0916             return '<TagAuthorization %s %s %s %s>' % (self.tag_name, self.access_type, self.credential, self.credential_type)
0917
0918         def to_array(self):
0919             return [self.tag_name, self.access_type, self.credential, self.credential_type]
0920
0921         def all(self, **kwargs):
0922             """
0923             Returns `amount` Records ordered by Record record.
0924             """
0925             query = self.session.query(TagAuthorization)
0926             query = apply_filters(query, TagAuthorization, kwargs)
0927             amount = kwargs["amount"] if "amount" in list(kwargs.keys()) else None
0928             query_result = query.order_by(TagAuthorization.tag).limit(amount).all()
0929             return data_sources.json_data_node.make(query_result)
0930
0931     classes = {"globaltag" : GlobalTag, "iov" : IOV, "globaltagmap" : GlobalTagMap,\
0932                 "payload" : Payload, "tag" : Tag, "TagAuthorization": TagAuthorization, "Base" : Base}
0933
0934     if class_name == None:
0935         return classes
0936     else:
0937         return classes[class_name]