Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2021-02-14 14:32:01

0001 #!/usr/bin/env perl
0002 use File::Basename;
0003 use lib dirname($0);
0004 use SCRAMGenUtils;
0005 use Getopt::Long;
0006 $|=1;
0007 
0008 my $pwd=`/bin/pwd`; chomp $pwd; $pwd=&SCRAMGenUtils::fixPath($pwd);
0009 my $defaultRuleFile="Utilities/ReleaseScripts/scripts/CMSSWReleaseDependency.rules";
0010 
0011 if(&GetOptions(
0012            "--release=s",\$release,
0013            "--rules=s",\$ruleFile,
0014            "--package=s",\$package,
0015            "--product=s",\$product,
0016            "--dependency=s",\$dependency,
0017            "--detail",\$detail,
0018            "--ignore-test",\$ignore_test,
0019            "--source-only",\$source_only,
0020            "--dump",\$dump,
0021            "--help",\$help,
0022               ) eq ""){print STDERR "#Wrong arguments.\n"; &usage_msg();}
0023 
0024 
0025 if (defined $help){&usage_msg();}
0026 
0027 if (defined $detail){$detail=1;}
0028 else{$detail=0;}
0029 
0030 if (defined $dump){$dump=1;}
0031 else{$dump=0;}
0032 
0033 if (defined $ignore_test){$ignore_test='[^\/]+\/[^\/]+\/test\/';}
0034 else{$ignore_test='';}
0035 
0036 if (!defined $release){$release=$pwd;}
0037 $release=&SCRAMGenUtils::scramReleaseTop($release);
0038 if($release eq ""){print STDERR "ERROR: Please run this script from a SCRAM-based area or use --release <path> option.\n"; exit 1;}
0039 
0040 &SCRAMGenUtils::init ($release);
0041 my $arch=&SCRAMGenUtils::getScramArch();
0042 
0043 my $cache={}; my $cache_config={};
0044 $cache=&initCache($release,$arch);
0045 
0046 if ($dependency eq "")
0047 {
0048   if (!defined $ruleFile)
0049   {
0050     $ruleFile = "${release}/src/${defaultRuleFile}";
0051     if (!-f $ruleFile)
0052     {
0053       my $releaseTop = &SCRAMGenUtils::getFromEnvironmentFile("RELEASETOP",$release);
0054       $ruleFile = "${releaseTop}/src/${defaultRuleFile}";
0055     }
0056   }
0057   if (!-f $ruleFile){die "ERROR: Can not find Project dependency rules file: $ruleFile\n";}
0058   ($cache->{RULES},$cache_config)=&readRules($ruleFile);
0059   if ((!defined $package) && (exists $cache_config->{packages})){$package=join(",",%{$cache_config->{packages}});}
0060 }
0061 else
0062 {
0063   my $rules=[];
0064   foreach my $dep (split(",",$dependency))
0065   {
0066     $dep=~s/^\s*//o;$dep=~s/\s*$//o;
0067     my $add=0;
0068     if ($package ne ""){push @$rules,["${package}/.*",$dep];$add=1;}
0069     if ($product ne ""){push @$rules,[".+/${product}",$dep];$add=1;}
0070     if(!$add){push @$rules,[".+",$dep];}
0071   }
0072   $cache->{RULES}{pos}{"not-allowed-uses"}=$rules;
0073 }
0074 
0075 my %packs=();
0076 if (defined $product)
0077 {
0078   foreach my $prod (split(/,/,$product))
0079   {
0080     $prod=~s/\s+//g;
0081     if ($prod eq ""){next;}
0082     if (exists $cache->{DEPS}{$prod}){$packs{$cache->{DEPS}{$prod}{PACK}}{$prod}=1;}
0083   }
0084 }
0085 if (defined $package)
0086 {
0087   foreach my $pack (split(/,/,$package))
0088   {
0089     $pack=~s/\s+//g;
0090     if ($pack eq ""){next;}
0091     foreach my $p (keys %{$cache->{DEPS}})
0092     {
0093       my $pk = $cache->{DEPS}{$p}{PACK};
0094       if ($pk=~/^$pack(\/.+|)$/){$packs{$pk}{$p}=1;}
0095     }
0096   }
0097 }
0098 if ((!defined $product) && (!defined $package))
0099 {
0100   foreach my $p (keys %{$cache->{DEPS}})
0101   {
0102     if ($cache->{DEPS}{$p}{TOOL} eq "self"){$packs{$cache->{DEPS}{$p}{PACK}}{$p}=1;}
0103   }
0104 }
0105 
0106 if (scalar(keys %packs)==0){die "ERROR: There are no products to check for dependency.\n";}
0107 
0108 foreach my $pk (sort keys %packs)
0109 {
0110   foreach my $prod (sort keys %{$packs{$pk}}){&checkDependency($prod,$pk,$cache);}
0111 }
0112 
0113 if ($dump)
0114 {
0115   foreach my $type (keys %{$cache->{RULES}})
0116   {
0117     foreach my $sec (keys %{$cache->{RULES}{$type}})
0118     {
0119       print "$sec ($type)\n";
0120       foreach my $r (@{$cache->{RULES}{$type}{$sec}}){print "  ",$r->[0]," => ",$r->[1],"\n";}
0121     }
0122   }
0123 }
0124 
0125 sub usage_msg()
0126 {
0127   my $s=basename($0);
0128   print "Usage: $s [--rules <rule-file>] [--package <package>] [--product <product>]\n",
0129         "          [--dependency <tool|product>] [--release <path>] [--detail] [--help]\n",
0130         " Script apply the dependency rules provides via rule-file and print out any\n",
0131     " voilation of those rules in BuildFiles. One can use --package <package> and\n",
0132     " --dependency <tool|product> option to find out why <package> is being link\n",
0133     " against <tool|product>.\n";
0134   exit 0;
0135 }
0136 
0137 sub checkDependency()
0138 {
0139   my ($prod,$pack,$cache)=@_;
0140   my $fprod="${pack}/${prod}";
0141   if ($prod eq $pack) { $fprod=$pack;}
0142   my %rules=();
0143   foreach my $type ("pos","nag")
0144   {
0145     foreach my $sec (keys %{$cache->{RULES}{$type}})
0146     {
0147       $rules{$type}{$sec}=[];
0148       foreach my $r (@{$cache->{RULES}{$type}{$sec}})
0149       {
0150         foreach my $x (@$r)
0151         {
0152           if ($fprod=~/^$x$/){push @{$rules{$type}{$sec}},$r;last;}
0153         }
0154       }
0155     }
0156   }
0157   my @allowed=();
0158   my $msg=0;
0159   foreach my $dep (keys %{$cache->{DEPS}{$prod}{ALL}})
0160   {
0161     my $isTool=0;
0162     if (exists $cache->{TOOLS}{$dep})
0163     {
0164       $isTool=1;
0165       my $found="";
0166       foreach my $a (@allowed){if (exists $cache->{DEPS}{$a}{ALL}{$dep}){$found=$a;last;}}
0167       if ($found){push @allowed,$dep; next;}
0168     }
0169     my $stat=&applyFilter($fprod,$dep,\%rules,"allowed-uses");
0170     if ($stat == -1)
0171     {
0172       $stat=&applyFilter($dep,$fprod,\%rules,"allowed-usedby");
0173       if ($stat == -1)
0174       {
0175         $stat=&applyFilter($fprod,$dep,\%rules,"not-allowed-uses");
0176         if ($stat == -1)
0177     {
0178       $stat=&applyFilter($dep,$fprod,\%rules,"not-allowed-usedby");
0179     }
0180     if ($stat==1){$stat=0;}
0181     else{$stat=1;}
0182       }
0183     }
0184     if ($stat == 0)
0185     {
0186       if (!$msg){print ">> Checking dependency for $fprod\n"; $msg=1;}
0187       my $type="indirect";
0188       if (exists $cache->{DEPS}{$prod}{DIRECT}{$dep}){$type="direct"}
0189       print "  ****ERROR: Dependency violation ($type): $fprod $dep\n";
0190       if ($detail){&searchDeps($prod,$dep,$cache,"  ");}
0191     }
0192     elsif ($isTool){push @allowed,$dep;}
0193   }
0194   if ($msg){print ">> Done Checking dependency for $fprod\n";}
0195 }
0196 
0197 sub searchDeps()
0198 {
0199   my ($p,$d,$cache,$tab)=@_;
0200   print "${tab}*",&product2Package($p,$cache)," ($p)\n";
0201   if ($p eq $d){return;}
0202   if (exists $cache->{DEPS}{$p}{ALL}{$d})
0203   {
0204     foreach my $u (keys %{$cache->{DEPS}{$p}{DIRECT}})
0205     {
0206       if ($u eq $d){print "${tab}  *$u (",$cache->{PACKS}{$u}[0],")\n";}
0207       if (!exists $cache->{PACKS}{$u}){next;}
0208       $u=$cache->{PACKS}{$u}[0];
0209       if (exists $cache->{DEPS}{$u}{ALL}{$d}){&searchDeps($u,$d,$cache,"$tab  ");}
0210     }
0211   }
0212 }
0213 
0214 
0215 #############################################################
0216 #Check for dependnecy rules and returns
0217 #-1: If there is no rule matches this dependnecy
0218 #0:  If dependency in negated for this section
0219 #1: If dependency is positive for this section
0220 #############################################################
0221 
0222 sub applyFilter()
0223 {
0224   my ($left,$right,$rules,$sec)=@_;
0225   my $l=""; my $r="";
0226   foreach my $rx (@{$rules->{nag}{$sec}})
0227   {
0228     $l=$rx->[0]; $r=$rx->[1];
0229     if ($left=~/^$l$/)
0230     {
0231       if ($right=~/^$r$/){return 0;}    
0232     }
0233   }
0234   foreach my $rx (@{$rules->{pos}{$sec}})
0235   {
0236     $l=$rx->[0]; $r=$rx->[1];
0237     if ($left=~/^$l$/)
0238     {
0239       if ($right=~/^$r$/){return 1;}
0240     }
0241   }
0242   return -1;
0243 }
0244 
0245 #################################
0246 # Read Project Dependency Rules #
0247 #################################
0248 sub readRules ()
0249 {
0250   my ($file)=@_;
0251   my $cache={}; my $config={};
0252   foreach my $t ("pos","nag")
0253   {
0254     foreach my $x ("allowed","not-allowed")
0255     {
0256       foreach my $y ("uses","usedby"){$cache->{$t}{"${x}-${y}"}=[];}
0257     }
0258   }
0259   my $ref;
0260   open($ref,$file) || die "ERROR: Can not open file for reading: $file\n";
0261   my $sec="";
0262   while(my $l=<$ref>)
0263   {
0264     chomp $l;
0265     if ($l=~/^\s*(#.*|)$/o){next;}
0266     if ($l=~/^\s*\[\s*([^\s\]]+)\s*]\s*/o)
0267     {
0268       $sec=lc($1);
0269       if (($sec ne "config") && (!exists $cache->{pos}{$sec})){$cache->{pos}{$sec}=[];$cache->{nag}{$sec}=[];}
0270     }
0271     elsif ($sec ne "")
0272     {
0273       my ($pks,$deps)=split(":",$l,2);
0274       if ($sec eq "config")
0275       {
0276         $pks=~s/\s+//g; $deps=~s/\s+//g;
0277     if (!exists $config->{$pks}){$config->{$pks}={};}
0278     foreach my $p (split(/,/,$deps)){$config->{$pks}{$p}=1;}
0279     next;
0280       }
0281       my $type="pos";
0282       $pks=~s/^\s*//o; $pks=~s/\s*$//o;
0283       if ($pks=~/^!(.+)/o){$pks=$1; $type="nag";}
0284       foreach my $dep (split ",",$deps)
0285       {
0286         $dep=~s/^\s*//o;$dep=~s/\s*$//o;
0287     push @{$cache->{$type}{$sec}},[$pks,$dep];
0288       }
0289     }
0290   }
0291   close($ref);
0292   return ($cache, $config);
0293 }
0294 
0295 ##########################################################################
0296 # Read Tools and Project cache of all externals and SCRAM-based projects #
0297 ##########################################################################
0298 sub readBuildFileCache()
0299 {
0300   my ($release,$arch,$cache)=@_;
0301   $cache->{Caches}{TOOLS}=&SCRAMGenUtils::readCache("${release}/.SCRAM/${arch}/ToolCache.db.gz");
0302   foreach my $t (keys %{$cache->{Caches}{TOOLS}{SETUP}})
0303   {
0304     my $sbase="";
0305     if ($cache->{Caches}{TOOLS}{SETUP}{$t}{SCRAM_PROJECT} == 1)
0306     {
0307       my $bv=uc($t)."_BASE";
0308       $sbase=$cache->{Caches}{TOOLS}{SETUP}{$t}{$bv};
0309     }
0310     elsif ($t eq "self"){$sbase=$release;}
0311     if ($sbase ne "")
0312     {
0313       $cache->{Caches}{$t}=&SCRAMGenUtils::readCache("${sbase}/.SCRAM/${arch}/ProjectCache.db.gz");
0314       foreach my $d (keys %{$cache->{Caches}{$t}{BUILDTREE}})
0315       {
0316         if ($ignore_test && ($cache->{Caches}{$t}{BUILDTREE}{$d}{CLASS} eq "TEST")){next;}
0317         &readPkgInfo($d,$t,$cache);
0318       }
0319       if ($t eq "self")
0320       {
0321         my $releaseTop=&SCRAMGenUtils::getFromEnvironmentFile("RELEASETOP",$release);
0322     if ($releaseTop ne "")
0323     {
0324       $cache->{Caches}{$t}=&SCRAMGenUtils::readCache("${releaseTop}/.SCRAM/${arch}/ProjectCache.db.gz");
0325       foreach my $d (keys %{$cache->{Caches}{$t}{BUILDTREE}})
0326       {
0327         if ($ignore_test && ($cache->{Caches}{$t}{BUILDTREE}{$d}{CLASS} eq "TEST")){next;}
0328             &readPkgInfo($d,$t,$cache);
0329       }
0330     }
0331       }
0332       delete $cache->{Caches}{$t};
0333     }
0334     else{&readToolsInfo(lc($t),$cache);}
0335   }
0336   delete $cache->{Caches};
0337 }
0338 
0339 sub initCache()
0340 {
0341   my ($release,$arch)=@_;
0342   my $cache={};
0343   if (!defined $source_only){&readBuildFileCache($release,$arch,$cache);}
0344   &updateSourceDeps($cache);
0345   foreach my $p (keys %{$cache->{PRODS}}){&allDeps($p,$cache);}
0346   foreach my $k (keys %$cache){if ($k!~/^(DEPS|TOOLS|PACKS)$/){delete $cache->{$k};}}
0347   return $cache;
0348 }
0349 
0350 sub allDeps()
0351 {
0352   my ($prod,$cache)=@_;
0353   if(exists $cache->{DEPS}{$prod}){return;}
0354   $cache->{DEPS}{$prod}{ALL}={};
0355   $cache->{DEPS}{$prod}{DIRECT}={};
0356   $cache->{DEPS}{$prod}{PACK}=$cache->{PRODS}{$prod}{PACK};
0357   $cache->{DEPS}{$prod}{TOOL}=$cache->{PRODS}{$prod}{TOOL};
0358   foreach my $d (keys %{$cache->{PRODS}{$prod}{DEPS}})
0359   {
0360     if (!exists $cache->{PACKS}{$d})
0361     {
0362       if (!exists $cache->{WARNS}{$d})
0363       {
0364         print "WARNING: UNKNOWN PACKAGE $d (DEPS OF $prod)\n";
0365         $cache->{WARNS}{$d}=1;
0366       }
0367     }
0368     else
0369     {
0370       my $p=$cache->{PACKS}{$d}[0];
0371       &allDeps($p,$cache);
0372       $cache->{DEPS}{$prod}{ALL}{$d}=1;
0373       $cache->{DEPS}{$prod}{DIRECT}{$d}=1;
0374       foreach my $u (keys %{$cache->{DEPS}{$p}{ALL}}){$cache->{DEPS}{$prod}{ALL}{$u}=1;}
0375     }
0376   }
0377 }
0378 
0379 sub readPkgInfo ()
0380 {
0381   my ($d,$t,$cache)=@_;
0382   my $c=$cache->{Caches}{$t}{BUILDTREE}{$d};
0383   my $suffix=$c->{SUFFIX};
0384   if($suffix ne ""){return;}
0385   my $class=$c->{CLASS};
0386   my $name=$c->{NAME};
0387   my $c1=$c->{RAWDATA}{content};
0388   $d=~s/^LCG\///;
0389   if($class=~/^(LIBRARY|CLASSLIB|SEAL_PLATFORM)$/o){&addProd($t,$name,dirname($d),$cache,$c1);}
0390   elsif($class eq "PACKAGE"){&addProd($t,$name,$d,$cache,$c1);}
0391   elsif($class=~/^(TEST|BIN|PLUGINS|BINARY)$/o){&addProds($t,$d,$cache,$c1);}
0392   elsif($class=~/^(PYTHON|SUBSYSTEM|DATA_INSTALL|SCRIPTS|PROJECT|IVS)$/o){return;}
0393   else{print STDERR "WARNING: UNKNOW TYPE $class in $t/$d\n";}
0394 }
0395 
0396 sub addProd()
0397 {
0398   my ($tool,$prod,$pack,$cache,$c)=@_;
0399   if (exists $cache->{PRODS}{$prod}){return;}
0400   if (defined $c)
0401   {
0402     &initProd($cache,$prod,$tool,$pack);
0403     if(exists $c->{USE}){&addDirectDeps($c->{USE},$cache->{PRODS}{$prod}{DEPS},$cache);}
0404     if((exists $c->{EXPORT}) && (exists $c->{EXPORT}{USE})){&addDirectDeps($c->{EXPORT}{USE},$cache->{PRODS}{$prod}{DEPS},$cache);}
0405   }
0406 }
0407 
0408 sub addProds()
0409 {
0410   my ($tool,$pack,$cache,$c)=@_;
0411   if (exists $cache->{PACKS}{$pack}){return;}
0412   if (!defined $c){return;}
0413   my $cuse=[];
0414   if (exists $c->{USE}){$cuse=$c->{USE};}
0415   if (exists $c->{BUILDPRODUCTS})
0416   {
0417     foreach my $t (keys %{$c->{BUILDPRODUCTS}})
0418     {
0419       foreach my $p (keys %{$c->{BUILDPRODUCTS}{$t}})
0420       {
0421         my $c1=$c->{BUILDPRODUCTS}{$t}{$p}{content};
0422     &addProd($tool,$p,$pack,$cache,$c->{BUILDPRODUCTS}{$t}{$p}{content});
0423     &addDirectDeps($cuse,$cache->{PRODS}{$p}{DEPS},$cache);
0424       }
0425     }
0426   }
0427 }
0428 
0429 sub addDirectDeps()
0430 {
0431   my ($uses,$c,$cache)=@_;
0432   foreach my $u (@{$uses})
0433   {
0434     $u=~s/^LCG\///;
0435     my $p=$u;
0436     my $t=lc($u);
0437     if (exists $cache->{Caches}{TOOLS}{SETUP}{$t}){$p=$t;}
0438     $c->{$p}=1;
0439   }
0440 }
0441 
0442 sub readToolsInfo()
0443 {
0444   my ($t,$cache)=@_;
0445   if (exists $cache->{PRODS}{$t}){return;}
0446   $cache->{TOOLS}{$t}=1;
0447   $cache->{PRODS}{$t}{TOOL}=$t;
0448   $cache->{PRODS}{$t}{PACK}=$t;
0449   $cache->{PRODS}{$t}{DEPS}={};
0450   $cache->{PACKS}{$t}[0]=$t;
0451   if (exists $cache->{Caches}{TOOLS}{SETUP}{$t}{USE})
0452   {&addDirectDeps($cache->{Caches}{TOOLS}{SETUP}{$t}{USE},$cache->{PRODS}{$t}{DEPS},$cache);}
0453 }
0454 
0455 sub product2Package()
0456 {
0457   my ($p,$cache)=@_;
0458   if(exists $cache->{PROD2PACK}{$p}){return $cache->{PROD2PACK}{$p};}
0459   my $pk="";
0460   if (exists $cache->{DEPS}{$p}){$pk=$cache->{DEPS}{$p}{PACK};}
0461   $cache->{PROD2PACK}{$p}=$pk;
0462   return $pk;
0463 }
0464 
0465 sub updateSourceDeps ()
0466 {
0467   my ($cache)=@_;
0468   $cache->{FILES}={};
0469   if (!&readUses($cache,$ENV{CMSSW_BASE})){&readCompilerDeps($cache,$ENV{CMSSW_BASE},$ENV{CMSSW_RELEASE_BASE});}
0470   if ($ENV{CMSSW_RELEASE_BASE}){&readUses($cache,$ENV{CMSSW_RELEASE_BASE});}
0471   delete $cache->{FILES};
0472 }
0473 
0474 sub initProd ()
0475 {
0476   my ($cache,$prod,$tool,$pack)=@_;
0477   if (exists $cache->{PRODS}{$prod}){return;}
0478   $cache->{PRODS}{$prod}{TOOL}=$tool;
0479   $cache->{PRODS}{$prod}{PACK}=$pack;
0480   $cache->{PRODS}{$prod}{DEPS}={};
0481   if (!exists $cache->{PACKS}{$pack}){$cache->{PACKS}{$pack}=[];}
0482   push @{$cache->{PACKS}{$pack}},$prod;
0483 }
0484 
0485 sub readCompilerDeps()
0486 {
0487   my ($cache,$path,$rbase)=@_;
0488   if (!-d "${path}/tmp/$ENV{SCRAM_ARCH}/src"){return;}
0489   my $ref;
0490   foreach my $file (`find ${path}/tmp/$ENV{SCRAM_ARCH}/src -name '*.dep' -type f`)
0491   {
0492     chomp $file;
0493     if ($ignore_test && $file=~/\/src\/$ignore_test/){next;}
0494     my $srcpack=""; my $prod="";
0495     open($ref,$file) || die "Can not open file for reading: $file\n";
0496     while(my $line=<$ref>)
0497     {
0498       chomp $line;
0499       if ($line=~/:/o){$prod=""; next;}
0500       if ($line=~/^\s*$/o){next;}
0501       if ($rbase){$line=~s/^\s*${rbase}\///o;}
0502       if ($line=~/^\//){next;}
0503       $line=~s/^\s*src\///o; $line=~s/\s*\\//o;
0504       my $pack=$line; $pack=~s/([^\/]+\/[^\/]+)\/.+/$1/;
0505       if ($prod eq "")
0506       {
0507         $srcpack=$pack;
0508         $prod=$line;
0509         $cache->{FILES}{$prod}=1;
0510         &initProd($cache,$prod,"self",$prod);
0511         next;
0512       }
0513       elsif ($srcpack ne $pack){&initProd($cache,$line,"self",$line); $cache->{PRODS}{$prod}{DEPS}{$line}=1;}
0514     }
0515     close($ref);
0516   }
0517 }
0518 
0519 sub readUses()
0520 {
0521   my ($cache,$path)=@_;
0522   my $uses="${path}/etc/dependencies/uses.out.gz";
0523   if (-f $uses)
0524   {
0525     my $ref;
0526     open($ref,"gunzip -c $uses |") || die "Can not open file for reading: $uses\n";
0527     while  (my $line=<$ref>)
0528     { 
0529       chomp($line);
0530       my ($src,$line)=split(' ',$line,2);
0531       if ($src ne "")
0532       {
0533         if ($ignore_test && $src=~/^$ignore_test/){next;}
0534         if (exists $cache->{FILES}{$src}){next;}
0535         my $pack=$src;
0536         $pack=~s/([^\/]+\/[^\/]+)\/.+/$1/;
0537         my $prod=$src;
0538         &initProd($cache,$prod,"self",$prod);
0539         foreach my $p (split(' ',$line))
0540         {
0541           my $x=$p;
0542           $p=~s/([^\/]+\/[^\/]+)\/.+/$1/;
0543           if ($p ne $pack){&initProd($cache,$x,"self",$x); $cache->{PRODS}{$prod}{DEPS}{$x}=1;}
0544         }
0545       }
0546     }
0547     close($ref);
0548     return 1;
0549   }
0550   return 0;
0551 }