XMLIdealGeometryESSource/test/runXMLBigFileToDBAndBackValidation.sh

0001 #!/bin/tcsh
0002 # Michael Case 2009-09-17:  This is meant to validate that the geometry going into the db is
0003 # indeed the same as the geometry that comes from the many xml files which created the db.
0004 # It does NOT check SpecPars (yet).
0005 # It checks the position of all parts in the hierarchy of the graph of parts positioned
0006 # in the detector and is currently (in the file testCompareDumpFiles.py) set to look
0007 # for differences exceeting .0004 mm in x, y and z and .0004 in the elements of the
0008 # rotation matrix.
0009 #
0010 # To run this file, ./runXMLBigFileToDBAndBackValidation.sh in
0011 # GeometryReaders/XMLIdealGeometryESSource/test
0012 # To RERUN the test, rm -rf workarea.
0013
0014 # What I want to know is 1 where the config file is located (for domcount and dddreport)
0015 # and 2 what the sub-string corresponding to that is in the Configuration/Geometry.
0016 cmsenv
0017 if ($#argv == 0) then
0018    setenv geometry "GeometryIdeal"
0019 else
0020    setenv geometry `echo ${1}`
0021 endif
0022
0023 set geomtemp = `(grep "Geometry.CMSCommonData" ${CMSSW_RELEASE_BASE}/src/Configuration/Geometry/python/${geometry}_cff.py | awk 'split($2,a,"."){print a[3]}')`
0024 #awk -F\. '{print $3}')`
0025 set geomxml = "${CMSSW_RELEASE_BASE}/src/Geometry/CMSCommonData/python/${geomtemp}.py"
0026
0027 echo "START - All messages in this script pertain to geometry data described in Configuration/Geometry/python/${geometry}_cff.py"
0028 echo "        and xml files in: ${geomxml}"
0029
0030 # STEP 1:
0031 # validate current set of xml files in $geomxml is valid
0032 #ASSUMPTIONS:  1.  relative path between documents (xml) and schema (DetectorDescription/Schema/DDLSchema.xsd)
0033 #                  are no more than 4 away, i.e. ../../../../ MAX (for normal/cmsextent.xml files)
0034 #grep ".xml" $geomxml | sed "{s/'//g}" | sed '{s/,//g}' | sed '{s/ //g}' | sed '{s/\t//g}' | sed '{s/geomXMLFiles=cms.vstring(//g}' | sed '{s/)//g}' | grep -v "#" >! /tmp/tmpcmsswdddxmlfileslist
0035 set whst=`(grep ".xml" $geomxml | sed "{s/'//g}" | sed '{s/,//g}' | sed '{s/ //g}' | sed '{s/\t//g}' | sed '{s/geomXMLFiles=cms.vstring(//g}'  | sed '{s/+cms.vstring(//g}' | sed '{s/)//g}' | grep -v "#" )`
0036 #echo $whst
0037 mkdir workarea
0038 #rm -f dcorig.out
0039 touch dcorig.out
0040 #set the schema path
0041 if ( -e "${CMSSW_BASE}/src/DetectorDescription/Schema/DDLSchema.xsd" ) then
0042     set schpath = `(echo "file://${CMSSW_BASE}/src/DetectorDescription/Schema/DDLSchema.xsd")`
0043 else
0044     set schpath = `(echo "file://${CMSSW_RELEASE_BASE}/src/DetectorDescription/Schema/DDLSchema.xsd")`
0045 endif
0046 echo "Assuming the schema is here: " $schpath
0047 #prep schpath for feeding into sed.
0048 set schpath = `(echo $schpath | sed '{s/\//\\\//g}')`
0049
0050     foreach l ( $whst )
0051         if ( -e $CMSSW_BASE/src/$l ) then
0052             set dp = `(echo "${l}" | awk -F\/ '{print NF}')`
0053             set fn = `(echo "${l}" | awk -F\/ '{print $NF}')`
0054             cp $CMSSW_BASE/src/$l .
0055             if ( $dp > 8 ) then
0056                 echo "ERROR: file " $fn " has a relative path too big for this script."
0057                 echo File with problems is $l
0058             else
0059                 sed -i "{s/..\/..\/..\/..\/DetectorDescription\/Schema\/DDLSchema.xsd/${schpath}/g}" $fn
0060             endif
0061             sed -i "{s/..\/..\/..\/DetectorDescription\/Schema\/DDLSchema.xsd/${schpath}/}" $fn
0062             DOMCount -v=always -n -s -f $fn >>& dcorig.out
0063             rm -f $fn
0064         else
0065             if ( -e $CMSSW_RELEASE_BASE/src/$l ) then
0066                 set dp = `(echo "${l}" | awk -F\/ '{print NF}')`
0067                 set fn = `(echo "${l}" | awk -F\/ '{print $NF}')`
0068                 cp $CMSSW_RELEASE_BASE/src/$l .
0069                 if ( $dp > 8 ) then
0070                     echo "ERROR: file " $fn " has a relative path too big for this script."
0071                     echo File with problems is $l
0072                 else
0073                     sed -i "{s/..\/..\/..\/..\/DetectorDescription\/Schema\/DDLSchema.xsd/${schpath}/g}" $fn
0074                 endif
0075                 sed -i "{s/..\/..\/..\/DetectorDescription\/Schema\/DDLSchema.xsd/${schpath}/}" $fn
0076                 DOMCount -v=always -n -s -f $fn >>& dcorig.out
0077                 rm -f $fn
0078             else
0079                 echo "ERROR: file " $l " not found in " $CMSSW_RELEASE_BASE "/src or " $CMSSW_BASE "/src" >>& dcorig.out
0080             endif
0081         endif
0082     end
0083     set errcnt = `(grep --count "Error" dcorig.out)`
0084     set warcnt = `(grep --count "Warning" dcorig.out)`
0085     if ($errcnt != 0 || $warcnt != 0) then
0086         echo "WARNING: There ARE XML Schema violations in original XML files and can be seen in dcorig.out."
0087     else
0088         echo "There ARE NO XML Schema violations in original XML files."
0089     endif
0090 #else
0091 #    echo "Missing ../../../DetectorDescription/Schema/DDLSchema.xsd..."
0092 #    echo "If you are running in your own work area, please check out (addpkg) DetectorDescription/Schema in your src directory."
0093 #    echo "ERROR: DOMCount validation not performed... others might still work below."
0094 #endif
0095
0096 cd workarea
0097
0098 # STEP 2:
0099 # validate current ddd model has no missing solids, materials or logical parts
0100 #dddreport.sh >& dddreport.out
0101 ../testdddreport.sh ${geomtemp}.py >& dddreport.out
0102 set whst=`(grep -n "Start checking" dddreport.out | awk -F : '{print $1}')`
0103 set totsiz=`(wc -l dddreport.out | awk '{print $1}')`
0104 @ tsdif = $totsiz - $whst
0105 #set tsdif = "${totsiz} - ${whst}"
0106 #echo "GOT HERE " $totsiz " - " $whst
0107 tail -$tsdif dddreport.out >& dddreptail.out
0108 set diffout = `(diff dddreptail.out ../dddreptail.ref)`
0109 if ( "$diffout" != "") then
0110     echo "WARNING: There ARE differences in the DD named objects from the standard xml files since the last ddreport.sh was run."
0111     echo "WARNING: Please check workarea/dddreport.out and workarea/dddreptail.out."
0112 else
0113     echo "There ARE NO differences in the DD named objects from the standard xml files since the last ddreport.sh was run."
0114 endif
0115
0116 mkdir db
0117 mkdir xml
0118 cd db
0119 # STEP 3: prepare database, prepare XML file to be loaded in DB.
0120 # The rm lines can be removed for debugging to check what is going on.
0121 #rm -f myfile.db
0122 #rm -f trXMLFromDB.out
0123 #rm -f twLoadDBWithXML.out
0124 #rm -f *.log.xml
0125 #rm -f *.log
0126 #rm -f dumpBDB
0127 #rm -f dumpSpecsdumpBDB
0128 echo "Start to write the single BIG XML file."
0129 # At this point I'm writing the XML file, 'fred.xml'
0130 # ASSUMPTION:  1) In the file CondTools/Geometry/test/geometryxmlwriter.py there will be always GeometryExtended
0131 #                 in the name of the config to be loaded. IF NOT, let me know and I'll adjust this (Mike Case)
0132 touch twLoadDBWithXML.out
0133 cp $CMSSW_RELEASE_BASE/src/CondTools/Geometry/test/geometryxmlwriter.py .
0134 sed -i "{s/GeometryExtended/${geometry}/}" geometryxmlwriter.py >>& twLoadDBWithXML.out
0135 sed -i "{s/geTagXX/fred/g}" geometryxmlwriter.py
0136 #sed -i '{s/GeometryExtended/GeometryIdeal/}' geometryxmlwriter.py >> twLoadDBWithXML.out
0137 cmsRun geometryxmlwriter.py >>& twLoadDBWithXML.out
0138 echo "Finish the write to the single BIG XML file."
0139
0140 # STEP 4:
0141 # make sure fred.xml has appropriate relative path to DDLSchema.xsd
0142
0143 #sed -i '{s/..\/..\/..\/D/..\/..\/..\/..\/..\/D/g}' fred.xml
0144 sed -i "{s/..\/..\/..\/DetectorDescription\/Schema\/DDLSchema.xsd/${schpath}/}" fred.xml
0145 DOMCount -n -s -f -v=always fred.xml >& dcBig.out
0146 #set diffout = `(diff diffdom.out ../../domcountBIG.ref)`
0147 set errcnt = `(grep --count "Error" dcBig.out)`
0148 set warcnt = `(grep --count "Warning" dcBig.out)`
0149 #if ( "$diffout" != "" ) then
0150 if ($errcnt != 0 || $warcnt != 0) then
0151     echo "WARNING: There ARE Schema violations in the single BIG XML file."
0152 else
0153     echo "There ARE NO Schema violations in the single BIG XML file."
0154 endif
0155 # validate current ddd model AS TRANSFERRED TO THE BIG XML FILE has no missing solids, materials or logical parts
0156 grep -v "File" $CMSSW_BASE/src/dddreportconfig.xml >& dddbigfilereport.xml
0157 sed -i '{s/<Root/<File name="GeometryReaders\/XMLIdealGeometryESSource\/test\/workarea\/db\/fred\.xml" url="\."\/><Root/}' dddbigfilereport.xml
0158 DDErrorReport GeometryReaders/XMLIdealGeometryESSource/test/workarea/db/dddbigfilereport.xml >& dddreport.out
0159 set whst=`(grep -n "Start checking" dddreport.out | awk -F : '{print $1}')`
0160 set totsiz=`(wc -l dddreport.out | awk '{print $1}')`
0161 #echo "GOT HERE " $totsiz " - " $whst
0162 @ tsdif2 = $totsiz - $whst
0163 tail -$tsdif2 dddreport.out >& dddreptail.out
0164 set diffout = `(diff dddreptail.out ../../dddreptail.ref)`
0165 if ( "$diffout" != "" ) then
0166     echo "WARNING: There ARE differences in the DD named objects from the single BIG xml file since the last ddreport.sh was run.  Please check dddreptail.out."
0167 else
0168     echo "There ARE NO differences in the DD named objects from the single BIG xml file since the last ddreport.sh was run."
0169 endif
0170
0171 # STEP 5
0172 echo "Start to write all geometry objects to the local DB including BIG XML file."
0173
0174 # At this point I'm writing ALL geometries, not only the "big file" into the database.
0175 cp $CMSSW_RELEASE_BASE/src/CondTools/Geometry/test/geometrywriter.py .
0176 sed -i "{s/GeometryExtended/${geometry}/}" geometrywriter.py >>& twLoadDBWithXML.out
0177 sed -i "{s/geTagXX/fred/g}" geometrywriter.py >>& twLoadDBWithXML.out
0178 #sed -i '{s/GeometryExtended/GeometryIdeal/}' geometrywriter.py >> twLoadDBWithXML.out
0179 cmsRun geometrywriter.py >>& twLoadDBWithXML.out
0180 echo "Finish writing all geometry objects to the local DB including BIG XML file."
0181 echo "Start to read the big XML file FROM the DB object"
0182 cmsRun ../../testReadXMLFromDB.py >& trXMLFromDB.out
0183 echo "Done with reading the big XML file FROM the DB object"
0184 cd ../xml
0185 #uncomment for debugging.
0186 #rm -f trIdeal.out
0187 #rm -f dumpSTD
0188 #rm -f dumpSpecdumpSTD
0189 #rm -f diffgeomIdeal.out
0190 echo "Start reading the XML from the original config file."
0191 cp ../../readExtendedAndDump.py .
0192 sed -i "{s/GeometryExtended/${geometry}/}" readExtendedAndDump.py >& trIdeal.out
0193 cmsRun readExtendedAndDump.py >>& trIdeal.out
0194 echo "End reading the XML from the original config file."
0195 cd ../..
0196 cmsRun testCompareDumpFiles.py >& tcdf.out
0197 # validate geometry from xml files to db.
0198 set wccnt = `(wc -l tcdf.out | awk '{print $1}')`
0199 if ( $wccnt == 0 ) then
0200     echo "All differences in position are less than tolerance."
0201 else
0202     echo "WARNING: There are $wccnt lines with differences greater than tolerance.  Please check tcdf.out for differences."
0203     echo "WARNING: Tolerance can be changed in the file testCompareDumpFiles.py."
0204 endif
0205
0206 echo "ALL DONE!"
0207
0208