File indexing completed on 2024-11-26 02:34:35
0001
0002 """
0003 The script compares two ROOT files and fills specified database file with
0004 comparison information.
0005
0006 Author: Albertas Gimbutas, Vilnius University (LT)
0007 e-mail: albertasgim@gmail.com
0008
0009 Note: balcklist support not implemented.
0010 """
0011 import sys
0012 import sqlite3
0013 from datetime import datetime
0014 from multiprocessing import Pool, Queue, Process
0015 from optparse import OptionParser, OptionGroup
0016 from os import makedirs
0017 from os.path import basename, join, exists
0018 from optparse import OptionParser
0019
0020 from Utilities.RelMon.utils_v2 import ComparisonError, tests, init_database, get_version
0021 from Utilities.RelMon.web.app_utils import get_release, get_stats, get_percentage, get_img_url, get_dataset_name
0022
0023
0024 parser = OptionParser(usage='Usage: %prog <file1> <file2> --db DB_NAME [options]')
0025 parser.add_option('--dir', action='store', dest='dir', default='.',
0026 help='Directory to store static html and .db file.')
0027 parser.add_option('--db', action='store', dest='db_name', default=None, help='path to SQLite3 database file.')
0028 parser.add_option('--st_test', action='store', dest='st_test', default='KS',
0029 help='Statistical test to use for the comparison.')
0030 parser.add_option('--th', action='store', dest='threshold', default=1e-5,
0031 help='Threshold to use in static HTML. Default: %default.')
0032 parser.add_option('--html', action='store_true', dest='html', default=False,
0033 help='Generate static html. Default: %default.')
0034 parser.add_option('--cl', action='store_true', dest='clear_db', default=False,
0035 help='Clear database before using.')
0036
0037 class RootFileComparison(object):
0038 def __init__(self, db_name, work_path=None, do_html=False):
0039 self.db_name = db_name
0040 self.work_path = work_path
0041 self.do_html = do_html
0042
0043 def walk_through(self, c, directory, f1, f2, st_test, parent_id=None, path=''):
0044 c.execute('''INSERT INTO Directory(name, parent_id) VALUES (?, ?)''',
0045 (directory.GetName(), parent_id))
0046 dir_id = c.lastrowid
0047 from_id, till_id = None, None
0048 for elem in directory.GetListOfKeys():
0049 elem_name = elem.GetName()
0050 subdir = directory.Get(elem_name)
0051 if subdir:
0052 if subdir.IsFolder():
0053 subdir_from_id, subdir_till_id, subdir_id = self.walk_through(c, subdir,
0054 f1, f2, st_test, dir_id, path=join(path, elem_name))
0055 if subdir_till_id and (not till_id or subdir_till_id > till_id):
0056 till_id = subdir_till_id
0057 if subdir_from_id and (not from_id or subdir_from_id < from_id):
0058 from_id = subdir_from_id
0059 else:
0060 hist1 = f1.Get(join(directory.GetPath(), elem_name))
0061 hist2 = f2.Get(join(directory.GetPath(), elem_name))
0062 try:
0063 p_value = st_test.do_test(hist1, hist2)
0064 c.execute('''INSERT INTO HistogramComparison(name, p_value, directory_id)
0065 VALUES (?, ?, ?)''', (elem_name, p_value, dir_id))
0066 comp_id = c.lastrowid
0067 if comp_id > till_id:
0068 till_id = comp_id
0069 if not from_id or comp_id < from_id:
0070 from_id = comp_id
0071 except ComparisonError as e:
0072 print('Error comparing %s: %s' % (hist1, e))
0073 if from_id and till_id:
0074 c.execute('''UPDATE Directory SET from_histogram_id=?, till_histogram_id=?
0075 WHERE id=?''', (from_id, till_id, dir_id))
0076 return from_id, till_id, dir_id
0077
0078 def compare(self, filename1, filename2, st_test):
0079 if not 'TFile' in globals():
0080 from ROOT import TFile
0081 f1 = TFile(filename1)
0082 f2 = TFile(filename2)
0083
0084 conn = sqlite3.connect(self.db_name)
0085 c = conn.cursor()
0086
0087
0088 dir_DQMData = f1.GetDirectory("DQMData")
0089 dir_Run = None
0090 for elem in dir_DQMData.GetListOfKeys():
0091 elem_name = elem.GetName()
0092 if elem_name.startswith('Run '):
0093 dir_Run = dir_DQMData.Get(elem_name)
0094
0095 fid, tid, dir_id = self.walk_through(c, dir_Run, f1, f2, st_test)
0096
0097 c.execute('''DELETE FROM Directory WHERE from_histogram_id IS NULL
0098 AND till_histogram_id IS NULL''')
0099 c.execute('''INSERT INTO RootFileComparison(filename1, filename2, directory_id)
0100 VALUES (?, ?, ?)''', (basename(filename1), basename(filename2), dir_id))
0101 root_file_comparison_id = c.lastrowid
0102
0103 conn.commit()
0104 conn.close()
0105 f1.Close()
0106 f2.Close()
0107 return root_file_comparison_id
0108
0109 def was_compared(self, filename1, filename2, st_test_name):
0110 conn = sqlite3.connect(self.db_name)
0111 c = conn.cursor()
0112 c.execute('''SELECT release_comparison_id FROM RootFileComparison WHERE (filename1=? and filename2=?)
0113 OR (filename1=? and filename2=?)''', (filename1, filename2, filename2, filename1))
0114 file_comparison = c.fetchall()
0115
0116 for release_comparison_id in file_comparison:
0117 c.execute('''SELECT statistical_test FROM ReleaseComparison WHERE
0118 id = ?''', release_comparison_id)
0119 statistical_test = c.fetchone()
0120 if statistical_test and statistical_test[0] == st_test_name:
0121 conn.close()
0122 return True
0123 conn.close()
0124 return False
0125
0126
0127 if __name__ == '__main__':
0128 opts, args = parser.parse_args()
0129 if len(args) != 2:
0130 parser.error('Specify two files to use for the comparison.')
0131 if not opts.db_name:
0132 parser.error('Specify SQLite3 database file for the comparison.')
0133
0134 if opts.clear_db:
0135 print('Clearing DB: %s...' % opts.db_name, end=' ')
0136 open(opts.db_name, 'w').close()
0137 print('Done.')
0138
0139 init_database(opts.db_name)
0140
0141 print('Comparing files:\n%s\n%s\n' % (basename(args[0]), basename(args[1])))
0142 file_cmp = RootFileComparison(opts.db_name, opts.dir)
0143 file_cmp.compare(args[0], args[1], tests[opts.st_test]())
0144
0145 if opts.html:
0146 dbfile2html(opts.db_name, opts.threshold)
