Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2021-07-13 03:16:14

0001 #! /usr/bin/env python3
0002 """
0003 The script compares two ROOT files and fills specified database file with
0004 comparison information.
0005 
0006 Author:  Albertas Gimbutas,  Vilnius University (LT)
0007 e-mail:  albertasgim@gmail.com
0008 
0009 Note: balcklist support not implemented.
0010 """
0011 from __future__ import print_function
0012 import sys
0013 import sqlite3
0014 from datetime import datetime
0015 from multiprocessing import Pool, Queue, Process
0016 from optparse import OptionParser, OptionGroup
0017 from os import makedirs
0018 from os.path import basename, join, exists
0019 from optparse import OptionParser
0020 
0021 from Utilities.RelMon.utils_v2 import ComparisonError, tests, init_database, get_version
0022 from Utilities.RelMon.web.app_utils import get_release, get_stats, get_percentage, get_img_url, get_dataset_name
0023 
0024 
0025 parser = OptionParser(usage='Usage: %prog <file1> <file2> --db DB_NAME [options]')
0026 parser.add_option('--dir', action='store', dest='dir', default='.',
0027         help='Directory to store static html and .db file.')
0028 parser.add_option('--db', action='store', dest='db_name', default=None, help='path to SQLite3 database file.')
0029 parser.add_option('--st_test', action='store', dest='st_test', default='KS',
0030         help='Statistical test to use for the comparison.')
0031 parser.add_option('--th', action='store', dest='threshold', default=1e-5,
0032         help='Threshold to use in static HTML. Default: %default.')
0033 parser.add_option('--html', action='store_true', dest='html', default=False,
0034         help='Generate static html. Default: %default.')
0035 parser.add_option('--cl', action='store_true', dest='clear_db', default=False,
0036         help='Clear database before using.')
0037 
0038 class RootFileComparison(object):
0039     def __init__(self, db_name, work_path=None, do_html=False):
0040         self.db_name = db_name
0041         self.work_path = work_path
0042         self.do_html = do_html
0043 
0044     def walk_through(self, c, directory, f1, f2, st_test, parent_id=None, path=''):
0045         c.execute('''INSERT INTO Directory(name, parent_id) VALUES (?, ?)''',
0046                      (directory.GetName(), parent_id))
0047         dir_id = c.lastrowid
0048         from_id, till_id = None, None
0049         for elem in directory.GetListOfKeys():
0050             elem_name = elem.GetName()
0051             subdir = directory.Get(elem_name)
0052             if subdir:
0053                 if subdir.IsFolder():
0054                     subdir_from_id, subdir_till_id, subdir_id = self.walk_through(c, subdir,
0055                                             f1, f2, st_test, dir_id, path=join(path, elem_name))
0056                     if subdir_till_id and (not till_id or subdir_till_id > till_id):
0057                         till_id = subdir_till_id
0058                     if subdir_from_id and (not from_id or subdir_from_id < from_id):
0059                         from_id = subdir_from_id
0060                 else:
0061                     hist1 = f1.Get(join(directory.GetPath(), elem_name))
0062                     hist2 = f2.Get(join(directory.GetPath(), elem_name))
0063                     try:
0064                         p_value = st_test.do_test(hist1, hist2)
0065                         c.execute('''INSERT INTO HistogramComparison(name, p_value, directory_id)
0066                                      VALUES (?, ?, ?)''', (elem_name, p_value, dir_id))
0067                         comp_id = c.lastrowid
0068                         if comp_id > till_id:
0069                             till_id = comp_id
0070                         if not from_id or comp_id < from_id:
0071                             from_id = comp_id
0072                     except ComparisonError as e:
0073                         print('Error comparing %s: %s' % (hist1, e))
0074         if from_id and till_id:
0075             c.execute('''UPDATE Directory SET from_histogram_id=?, till_histogram_id=?
0076                            WHERE id=?''', (from_id, till_id, dir_id))
0077         return from_id, till_id, dir_id
0078 
0079     def compare(self, filename1, filename2, st_test):
0080         if not 'TFile' in globals():
0081             from ROOT import TFile
0082         f1 = TFile(filename1)
0083         f2 = TFile(filename2)
0084 
0085         conn = sqlite3.connect(self.db_name)
0086         c = conn.cursor()
0087 
0088         ## Create Directory and HistogramComparison structure in the DB
0089         dir_DQMData = f1.GetDirectory("DQMData")
0090         dir_Run = None
0091         for elem in dir_DQMData.GetListOfKeys():
0092             elem_name = elem.GetName()
0093             if elem_name.startswith('Run '):
0094                 dir_Run = dir_DQMData.Get(elem_name)
0095 
0096         fid, tid, dir_id = self.walk_through(c, dir_Run, f1, f2, st_test)
0097 
0098         c.execute('''DELETE FROM Directory WHERE from_histogram_id IS NULL
0099                      AND till_histogram_id IS NULL''')
0100         c.execute('''INSERT INTO RootFileComparison(filename1, filename2, directory_id)
0101                      VALUES (?, ?, ?)''', (basename(filename1), basename(filename2), dir_id))
0102         root_file_comparison_id = c.lastrowid
0103 
0104         conn.commit()
0105         conn.close()
0106         f1.Close()
0107         f2.Close()
0108         return root_file_comparison_id
0109 
0110     def was_compared(self, filename1, filename2, st_test_name):
0111         conn = sqlite3.connect(self.db_name)
0112         c = conn.cursor()
0113         c.execute('''SELECT release_comparison_id FROM RootFileComparison WHERE (filename1=? and filename2=?)
0114                      OR (filename1=? and filename2=?)''', (filename1, filename2, filename2, filename1))
0115         file_comparison = c.fetchall()
0116 
0117         for release_comparison_id in file_comparison:
0118             c.execute('''SELECT statistical_test FROM ReleaseComparison WHERE
0119                          id = ?''', release_comparison_id)
0120             statistical_test = c.fetchone()
0121             if statistical_test and statistical_test[0] == st_test_name:
0122                 conn.close()
0123                 return True
0124         conn.close()
0125         return False
0126 
0127 
0128 if __name__ == '__main__':
0129     opts, args = parser.parse_args()
0130     if len(args) != 2:
0131         parser.error('Specify two files to use for the comparison.')
0132     if not opts.db_name:
0133         parser.error('Specify SQLite3 database file for the comparison.')
0134 
0135     if opts.clear_db:
0136         print('Clearing DB: %s...' % opts.db_name, end=' ')
0137         open(opts.db_name, 'w').close()
0138         print('Done.')
0139 
0140     init_database(opts.db_name)
0141 
0142     print('Comparing files:\n%s\n%s\n' % (basename(args[0]), basename(args[1])))
0143     file_cmp = RootFileComparison(opts.db_name, opts.dir)
0144     file_cmp.compare(args[0], args[1], tests[opts.st_test]())
0145 
0146     if opts.html:
0147         dbfile2html(opts.db_name, opts.threshold)