File indexing completed on 2024-04-06 12:31:49
0001
0002 """
0003 The script compares two ROOT files and fills specified database file with
0004 comparison information.
0005
0006 Author: Albertas Gimbutas, Vilnius University (LT)
0007 e-mail: albertasgim@gmail.com
0008
0009 Note: balcklist support not implemented.
0010 """
0011 from __future__ import print_function
0012 import sys
0013 import sqlite3
0014 from datetime import datetime
0015 from multiprocessing import Pool, Queue, Process
0016 from optparse import OptionParser, OptionGroup
0017 from os import makedirs
0018 from os.path import basename, join, exists
0019 from optparse import OptionParser
0020
0021 from Utilities.RelMon.utils_v2 import ComparisonError, tests, init_database, get_version
0022 from Utilities.RelMon.web.app_utils import get_release, get_stats, get_percentage, get_img_url, get_dataset_name
0023
0024
0025 parser = OptionParser(usage='Usage: %prog <file1> <file2> --db DB_NAME [options]')
0026 parser.add_option('--dir', action='store', dest='dir', default='.',
0027 help='Directory to store static html and .db file.')
0028 parser.add_option('--db', action='store', dest='db_name', default=None, help='path to SQLite3 database file.')
0029 parser.add_option('--st_test', action='store', dest='st_test', default='KS',
0030 help='Statistical test to use for the comparison.')
0031 parser.add_option('--th', action='store', dest='threshold', default=1e-5,
0032 help='Threshold to use in static HTML. Default: %default.')
0033 parser.add_option('--html', action='store_true', dest='html', default=False,
0034 help='Generate static html. Default: %default.')
0035 parser.add_option('--cl', action='store_true', dest='clear_db', default=False,
0036 help='Clear database before using.')
0037
0038 class RootFileComparison(object):
0039 def __init__(self, db_name, work_path=None, do_html=False):
0040 self.db_name = db_name
0041 self.work_path = work_path
0042 self.do_html = do_html
0043
0044 def walk_through(self, c, directory, f1, f2, st_test, parent_id=None, path=''):
0045 c.execute('''INSERT INTO Directory(name, parent_id) VALUES (?, ?)''',
0046 (directory.GetName(), parent_id))
0047 dir_id = c.lastrowid
0048 from_id, till_id = None, None
0049 for elem in directory.GetListOfKeys():
0050 elem_name = elem.GetName()
0051 subdir = directory.Get(elem_name)
0052 if subdir:
0053 if subdir.IsFolder():
0054 subdir_from_id, subdir_till_id, subdir_id = self.walk_through(c, subdir,
0055 f1, f2, st_test, dir_id, path=join(path, elem_name))
0056 if subdir_till_id and (not till_id or subdir_till_id > till_id):
0057 till_id = subdir_till_id
0058 if subdir_from_id and (not from_id or subdir_from_id < from_id):
0059 from_id = subdir_from_id
0060 else:
0061 hist1 = f1.Get(join(directory.GetPath(), elem_name))
0062 hist2 = f2.Get(join(directory.GetPath(), elem_name))
0063 try:
0064 p_value = st_test.do_test(hist1, hist2)
0065 c.execute('''INSERT INTO HistogramComparison(name, p_value, directory_id)
0066 VALUES (?, ?, ?)''', (elem_name, p_value, dir_id))
0067 comp_id = c.lastrowid
0068 if comp_id > till_id:
0069 till_id = comp_id
0070 if not from_id or comp_id < from_id:
0071 from_id = comp_id
0072 except ComparisonError as e:
0073 print('Error comparing %s: %s' % (hist1, e))
0074 if from_id and till_id:
0075 c.execute('''UPDATE Directory SET from_histogram_id=?, till_histogram_id=?
0076 WHERE id=?''', (from_id, till_id, dir_id))
0077 return from_id, till_id, dir_id
0078
0079 def compare(self, filename1, filename2, st_test):
0080 if not 'TFile' in globals():
0081 from ROOT import TFile
0082 f1 = TFile(filename1)
0083 f2 = TFile(filename2)
0084
0085 conn = sqlite3.connect(self.db_name)
0086 c = conn.cursor()
0087
0088
0089 dir_DQMData = f1.GetDirectory("DQMData")
0090 dir_Run = None
0091 for elem in dir_DQMData.GetListOfKeys():
0092 elem_name = elem.GetName()
0093 if elem_name.startswith('Run '):
0094 dir_Run = dir_DQMData.Get(elem_name)
0095
0096 fid, tid, dir_id = self.walk_through(c, dir_Run, f1, f2, st_test)
0097
0098 c.execute('''DELETE FROM Directory WHERE from_histogram_id IS NULL
0099 AND till_histogram_id IS NULL''')
0100 c.execute('''INSERT INTO RootFileComparison(filename1, filename2, directory_id)
0101 VALUES (?, ?, ?)''', (basename(filename1), basename(filename2), dir_id))
0102 root_file_comparison_id = c.lastrowid
0103
0104 conn.commit()
0105 conn.close()
0106 f1.Close()
0107 f2.Close()
0108 return root_file_comparison_id
0109
0110 def was_compared(self, filename1, filename2, st_test_name):
0111 conn = sqlite3.connect(self.db_name)
0112 c = conn.cursor()
0113 c.execute('''SELECT release_comparison_id FROM RootFileComparison WHERE (filename1=? and filename2=?)
0114 OR (filename1=? and filename2=?)''', (filename1, filename2, filename2, filename1))
0115 file_comparison = c.fetchall()
0116
0117 for release_comparison_id in file_comparison:
0118 c.execute('''SELECT statistical_test FROM ReleaseComparison WHERE
0119 id = ?''', release_comparison_id)
0120 statistical_test = c.fetchone()
0121 if statistical_test and statistical_test[0] == st_test_name:
0122 conn.close()
0123 return True
0124 conn.close()
0125 return False
0126
0127
0128 if __name__ == '__main__':
0129 opts, args = parser.parse_args()
0130 if len(args) != 2:
0131 parser.error('Specify two files to use for the comparison.')
0132 if not opts.db_name:
0133 parser.error('Specify SQLite3 database file for the comparison.')
0134
0135 if opts.clear_db:
0136 print('Clearing DB: %s...' % opts.db_name, end=' ')
0137 open(opts.db_name, 'w').close()
0138 print('Done.')
0139
0140 init_database(opts.db_name)
0141
0142 print('Comparing files:\n%s\n%s\n' % (basename(args[0]), basename(args[1])))
0143 file_cmp = RootFileComparison(opts.db_name, opts.dir)
0144 file_cmp.compare(args[0], args[1], tests[opts.st_test]())
0145
0146 if opts.html:
0147 dbfile2html(opts.db_name, opts.threshold)
